feat(asset): decouple logical assets from physical storage with StorageKey and implement O(1) copy

This commit is contained in:
yiling
2026-05-25 00:52:45 +08:00
parent c5fba746be
commit 5dc3bdb38e
33 changed files with 2203 additions and 273 deletions
@@ -0,0 +1,25 @@
/*
Warnings:
- You are about to drop the column `key` on the `assets` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "assets" DROP COLUMN "key",
ADD COLUMN "storage_key_id" TEXT;
-- CreateTable
CREATE TABLE "storage_keys" (
"id" TEXT NOT NULL,
"key" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "storage_keys_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "storage_keys_key_key" ON "storage_keys"("key");
-- AddForeignKey
ALTER TABLE "assets" ADD CONSTRAINT "assets_storage_key_id_fkey" FOREIGN KEY ("storage_key_id") REFERENCES "storage_keys"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+13 -1
View File
@@ -347,7 +347,6 @@ model Asset {
id String @id @default(ulid())
name String @default("")
nameNgram String[] @default([]) @map("name_ngram")
key String?
type AssetType
mediaType String? @map("media_type")
fileCount Int @default(0) @map("file_count")
@@ -370,6 +369,9 @@ model Asset {
target Asset? @relation("SymlinkTarget", fields: [targetId], references: [id])
symlinks Asset[] @relation("SymlinkTarget")
storageKeyId String? @map("storage_key_id")
storageKey StorageKey? @relation(fields: [storageKeyId], references: [id])
metadataValues AssetMetadataValue[] @relation("AssetToMetadataValues")
creatorId String? @map("creator_id")
@@ -403,6 +405,16 @@ model Asset {
@@unique([parentId, targetId], name: "parentIdTargetId")
}
model StorageKey {
id String @id @default(ulid())
key String @unique
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
assets Asset[]
@@map("storage_keys")
}
model AssetMetadataValue {
id String @id @default(ulid())
assetId String @map("asset_id")
+1 -1
View File
@@ -76,7 +76,7 @@ describe('readSkillTool', () => {
data: {
id: 'asset1',
name: 'skill.zip',
key: 'skills/skill.zip',
storageKey: { create: { key: 'skills/skill.zip' } },
type: 'file',
status: 'uploaded',
},
+3 -2
View File
@@ -65,15 +65,16 @@ export const createReadSkillTool = (
const asset = await prisma.asset.findUnique({
where: { id: skill.assetId },
include: { storageKey: true },
})
if (!asset || !asset.key) {
if (!asset || !asset.storageKey?.key) {
throw new Error('Skill asset not found or has no key')
}
const { buffer: zipBuffer } = await s3Service.getObject(
process.env.S3_BUCKET || 'shumai',
asset.key,
asset.storageKey.key,
)
const zip = new AdmZip(zipBuffer)
+5
View File
@@ -82,6 +82,11 @@ export type ShareLink = Prisma.ShareLinkModel
*
*/
export type Asset = Prisma.AssetModel
/**
* Model StorageKey
*
*/
export type StorageKey = Prisma.StorageKeyModel
/**
* Model AssetMetadataValue
*
+5
View File
@@ -106,6 +106,11 @@ export type ShareLink = Prisma.ShareLinkModel
*
*/
export type Asset = Prisma.AssetModel
/**
* Model StorageKey
*
*/
export type StorageKey = Prisma.StorageKeyModel
/**
* Model AssetMetadataValue
*
File diff suppressed because one or more lines are too long
+88 -2
View File
@@ -397,6 +397,7 @@ export const ModelName = {
ProjectMember: 'ProjectMember',
ShareLink: 'ShareLink',
Asset: 'Asset',
StorageKey: 'StorageKey',
AssetMetadataValue: 'AssetMetadataValue',
AssetComment: 'AssetComment',
AssetCommentAttachment: 'AssetCommentAttachment',
@@ -429,7 +430,7 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
omit: GlobalOmitOptions
}
meta: {
modelProps: "user" | "session" | "account" | "verification" | "authToken" | "team" | "sandbox" | "skill" | "teamMember" | "project" | "projectMember" | "shareLink" | "asset" | "assetMetadataValue" | "assetComment" | "assetCommentAttachment" | "assetEmbedding" | "invite" | "metadataField" | "notification" | "systemSettings" | "task" | "workflowTask" | "agent" | "agentSkill" | "agentSession" | "agentSessionEntry" | "userMetadata" | "model" | "provider"
modelProps: "user" | "session" | "account" | "verification" | "authToken" | "team" | "sandbox" | "skill" | "teamMember" | "project" | "projectMember" | "shareLink" | "asset" | "storageKey" | "assetMetadataValue" | "assetComment" | "assetCommentAttachment" | "assetEmbedding" | "invite" | "metadataField" | "notification" | "systemSettings" | "task" | "workflowTask" | "agent" | "agentSkill" | "agentSession" | "agentSessionEntry" | "userMetadata" | "model" | "provider"
txIsolationLevel: TransactionIsolationLevel
}
model: {
@@ -1395,6 +1396,80 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
}
}
}
StorageKey: {
payload: Prisma.$StorageKeyPayload<ExtArgs>
fields: Prisma.StorageKeyFieldRefs
operations: {
findUnique: {
args: Prisma.StorageKeyFindUniqueArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload> | null
}
findUniqueOrThrow: {
args: Prisma.StorageKeyFindUniqueOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
}
findFirst: {
args: Prisma.StorageKeyFindFirstArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload> | null
}
findFirstOrThrow: {
args: Prisma.StorageKeyFindFirstOrThrowArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
}
findMany: {
args: Prisma.StorageKeyFindManyArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>[]
}
create: {
args: Prisma.StorageKeyCreateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
}
createMany: {
args: Prisma.StorageKeyCreateManyArgs<ExtArgs>
result: BatchPayload
}
createManyAndReturn: {
args: Prisma.StorageKeyCreateManyAndReturnArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>[]
}
delete: {
args: Prisma.StorageKeyDeleteArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
}
update: {
args: Prisma.StorageKeyUpdateArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
}
deleteMany: {
args: Prisma.StorageKeyDeleteManyArgs<ExtArgs>
result: BatchPayload
}
updateMany: {
args: Prisma.StorageKeyUpdateManyArgs<ExtArgs>
result: BatchPayload
}
updateManyAndReturn: {
args: Prisma.StorageKeyUpdateManyAndReturnArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>[]
}
upsert: {
args: Prisma.StorageKeyUpsertArgs<ExtArgs>
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
}
aggregate: {
args: Prisma.StorageKeyAggregateArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.AggregateStorageKey>
}
groupBy: {
args: Prisma.StorageKeyGroupByArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.StorageKeyGroupByOutputType>[]
}
count: {
args: Prisma.StorageKeyCountArgs<ExtArgs>
result: runtime.Types.Utils.Optional<Prisma.StorageKeyCountAggregateOutputType> | number
}
}
}
AssetMetadataValue: {
payload: Prisma.$AssetMetadataValuePayload<ExtArgs>
fields: Prisma.AssetMetadataValueFieldRefs
@@ -2863,7 +2938,6 @@ export const AssetScalarFieldEnum = {
id: 'id',
name: 'name',
nameNgram: 'nameNgram',
key: 'key',
type: 'type',
mediaType: 'mediaType',
fileCount: 'fileCount',
@@ -2878,6 +2952,7 @@ export const AssetScalarFieldEnum = {
updatedAt: 'updatedAt',
parentId: 'parentId',
targetId: 'targetId',
storageKeyId: 'storageKeyId',
creatorId: 'creatorId',
taskId: 'taskId',
projectId: 'projectId'
@@ -2886,6 +2961,16 @@ export const AssetScalarFieldEnum = {
export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof AssetScalarFieldEnum]
export const StorageKeyScalarFieldEnum = {
id: 'id',
key: 'key',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
export type StorageKeyScalarFieldEnum = (typeof StorageKeyScalarFieldEnum)[keyof typeof StorageKeyScalarFieldEnum]
export const AssetMetadataValueScalarFieldEnum = {
id: 'id',
assetId: 'assetId',
@@ -3554,6 +3639,7 @@ export type GlobalOmitConfig = {
projectMember?: Prisma.ProjectMemberOmit
shareLink?: Prisma.ShareLinkOmit
asset?: Prisma.AssetOmit
storageKey?: Prisma.StorageKeyOmit
assetMetadataValue?: Prisma.AssetMetadataValueOmit
assetComment?: Prisma.AssetCommentOmit
assetCommentAttachment?: Prisma.AssetCommentAttachmentOmit
+12 -1
View File
@@ -64,6 +64,7 @@ export const ModelName = {
ProjectMember: 'ProjectMember',
ShareLink: 'ShareLink',
Asset: 'Asset',
StorageKey: 'StorageKey',
AssetMetadataValue: 'AssetMetadataValue',
AssetComment: 'AssetComment',
AssetCommentAttachment: 'AssetCommentAttachment',
@@ -270,7 +271,6 @@ export const AssetScalarFieldEnum = {
id: 'id',
name: 'name',
nameNgram: 'nameNgram',
key: 'key',
type: 'type',
mediaType: 'mediaType',
fileCount: 'fileCount',
@@ -285,6 +285,7 @@ export const AssetScalarFieldEnum = {
updatedAt: 'updatedAt',
parentId: 'parentId',
targetId: 'targetId',
storageKeyId: 'storageKeyId',
creatorId: 'creatorId',
taskId: 'taskId',
projectId: 'projectId'
@@ -293,6 +294,16 @@ export const AssetScalarFieldEnum = {
export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof AssetScalarFieldEnum]
export const StorageKeyScalarFieldEnum = {
id: 'id',
key: 'key',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
export type StorageKeyScalarFieldEnum = (typeof StorageKeyScalarFieldEnum)[keyof typeof StorageKeyScalarFieldEnum]
export const AssetMetadataValueScalarFieldEnum = {
id: 'id',
assetId: 'assetId',
+1
View File
@@ -21,6 +21,7 @@ export type * from './models/Project.ts'
export type * from './models/ProjectMember.ts'
export type * from './models/ShareLink.ts'
export type * from './models/Asset.ts'
export type * from './models/StorageKey.ts'
export type * from './models/AssetMetadataValue.ts'
export type * from './models/AssetComment.ts'
export type * from './models/AssetCommentAttachment.ts'
+386 -107
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+4 -2
View File
@@ -118,6 +118,7 @@ export class AiService {
const asset = await this.prismaClient.asset.findUnique({
where: { id: assetId },
include: { storageKey: true },
})
if (!asset) throw new Error('failed to get asset')
@@ -146,11 +147,12 @@ export class AiService {
throw new Error(`unsupported media type for embeddings: ${asset.mediaType}`)
}
if (!asset.key) {
const key = asset.storageKey?.key
if (!key) {
throw new Error(`asset has no key`)
}
const { buffer: data } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', asset.key)
const { buffer: data } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', key)
if (isImage) {
const embVec = await p.generateImageEmbedding(config.model, data)
+35 -16
View File
@@ -1293,25 +1293,39 @@ describe('AssetService', () => {
type: AssetType.file,
status: AssetStatus.pending_purge,
isDeleted: true,
projectId: project.id,
parentId: folderA.id,
key: `key-${i}`,
project: { connect: { id: project.id } },
parent: { connect: { id: folderA.id } },
storageKey: {
create: { key: `key-${i}` },
},
}))
await prisma.asset.createMany({ data: childData })
for (const data of childData) {
const asset = await prisma.asset.create({
data: data as any,
include: { storageKey: true },
})
await prisma.storageKey.update({
where: { id: asset.storageKeyId! },
data: { createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000) },
})
}
// 3. Trigger the purge job for the FIRST batch (100 items)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgePendingAssets()
// 4. Verify some items were processed in S3
// We don't know the order, but if Folder A was in the first 100,
// then its synchronous cascade will have wiped the remaining children.
// 5. Trigger the purge job for the SECOND batch
// ...
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgePendingAssets()
// 6. Trigger GC job to physically delete files
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgeUnreferencedStorageKeys()
// Call it again to process the remaining items (batch size is 100)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgeUnreferencedStorageKeys()
// 6. Verify S3 deletion was called for EVERY child
for (let i = 0; i < childCount; i++) {
expect(s3Service.deleteObject).toHaveBeenCalledWith(expect.any(String), `key-${i}`)
@@ -1334,24 +1348,29 @@ describe('AssetService', () => {
type: AssetType.file,
status: AssetStatus.pending_purge,
isDeleted: true,
projectId: project.id,
key: 'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
project: { connect: { id: project.id } },
storageKey: {
create: {
key: 'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000),
},
},
},
})
// Trigger the purge job
// 1. Purge the asset record
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgePendingAssets()
// 2. Trigger GC job
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgeUnreferencedStorageKeys()
// Verify deletePrefix was called with the correct prefix instead of just deleteObject
expect(s3Service.deletePrefix).toHaveBeenCalledWith(
expect.any(String),
'file/01KSBRJVY3DPK111S2MECFKDQ4/',
)
expect(s3Service.deleteObject).not.toHaveBeenCalledWith(
expect.any(String),
'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
)
// Verify record is gone
expect(await prisma.asset.findUnique({ where: { id: complexFile.id } })).toBeNull()
+134 -76
View File
@@ -18,7 +18,7 @@ import {
UpdateAssetOrderRequest,
UserInfo,
} from '@/dtos/asset'
import { Asset, AssetType, Prisma } from '@/generated/prisma/client.ts'
import { Asset, AssetType, Prisma, StorageKey } from '@/generated/prisma/client.ts'
import { PaginatedData, paginateQuery, PaginationParams } from '@/services/pagination'
import { s3Service } from '@/services/s3/s3'
import { generateKeyBetween } from 'jittered-fractional-indexing'
@@ -29,17 +29,19 @@ type AssetWithIncludes = Prisma.AssetGetPayload<{
include: {
creator: true
metadataValues: true
storageKey: true
target: {
include: {
creator: true
metadataValues: true
storageKey: true
children: {
include: { creator: true; metadataValues: true }
include: { creator: true; metadataValues: true; storageKey: true }
}
}
}
children: {
include: { creator: true; metadataValues: true }
include: { creator: true; metadataValues: true; storageKey: true }
}
}
}>
@@ -47,11 +49,11 @@ type AssetWithIncludes = Prisma.AssetGetPayload<{
type CommentWithIncludes = Prisma.AssetCommentGetPayload<{
include: {
creator: true
attachments: { include: { asset: true } }
attachments: { include: { asset: { include: { storageKey: true } } } }
replies: {
include: {
creator: true
attachments: { include: { asset: true } }
attachments: { include: { asset: { include: { storageKey: true } } } }
}
}
}
@@ -91,19 +93,21 @@ export class AssetService {
include: {
creator: true,
metadataValues: true,
storageKey: true,
target: {
include: {
creator: true,
metadataValues: true,
storageKey: true,
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
},
},
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
@@ -335,27 +339,17 @@ export class AssetService {
private async copyAssetRecursive(
tx: Prisma.TransactionClient,
asset: Asset,
asset: Asset & { storageKey?: StorageKey | null },
newParentId: string | null,
projectId: string | null,
sortIndex: string | null,
creatorId?: string,
withComments: boolean = false,
): Promise<Asset> {
let newKey = asset.key
if (asset.type === AssetType.file && asset.key) {
// Duplicate S3 object
const bucket = process.env.S3_BUCKET || 'shumai'
const ext = path.extname(asset.key)
newKey = ulid() + ext
await s3Service.copyObject(bucket, asset.key, bucket, newKey)
}
const newAsset = await tx.asset.create({
data: {
name: asset.name,
nameNgram: asset.nameNgram,
key: newKey,
type: asset.type,
mediaType: asset.mediaType,
fileCount: asset.fileCount,
@@ -372,6 +366,7 @@ export class AssetService {
taskId: asset.taskId,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
targetId: asset.targetId as any,
storageKeyId: asset.storageKeyId,
},
})
@@ -635,19 +630,21 @@ export class AssetService {
include: {
creator: true,
metadataValues: true,
storageKey: true,
target: {
include: {
creator: true,
metadataValues: true,
storageKey: true,
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
},
},
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
@@ -699,7 +696,14 @@ export class AssetService {
if (parent) data.parent = { connect: { id: parent.id } }
if (sortIndex) data.sortIndex = sortIndex
if (req.key) data.key = req.key
if (req.key) {
;(data as any).storageKey = {
connectOrCreate: {
where: { key: req.key },
create: { key: req.key },
},
}
}
if (req.sizeByte) data.sizeByte = req.sizeByte
if (req.contentType) data.mediaType = req.contentType
if (req.creatorId) data.creator = { connect: { id: req.creatorId } }
@@ -709,19 +713,21 @@ export class AssetService {
include: {
creator: true,
metadataValues: true,
storageKey: true,
target: {
include: {
creator: true,
metadataValues: true,
storageKey: true,
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
},
},
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
@@ -746,19 +752,21 @@ export class AssetService {
include: {
creator: true,
metadataValues: true,
storageKey: true,
target: {
include: {
creator: true,
metadataValues: true,
storageKey: true,
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
},
},
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
@@ -868,21 +876,23 @@ export class AssetService {
include: {
creator: true,
metadataValues: true,
storageKey: true,
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
},
},
children: {
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
},
storageKey: true,
},
})
const infos = await this.toAssetInfos([asset])
const infos = await this.toAssetInfos([asset as any])
return infos[0]
}
@@ -995,8 +1005,20 @@ export class AssetService {
setTimeout(runPurge, 1000) // Run every 1 second
}
// Stage 3: Garbage Collection Job - Purge unreferenced StorageKeys and physical files
const runGc = async () => {
if (!this.cleanupJobsRunning) return
try {
await this.purgeUnreferencedStorageKeys()
} catch (e: unknown) {
console.error('Error in asset GC job:', e)
}
setTimeout(runGc, 10000) // Run every 10 seconds
}
runExpiration()
runPurge()
runGc()
}
stopCleanupJob() {
@@ -1045,7 +1067,7 @@ export class AssetService {
private async purgePendingAssets() {
// 1. Lock 100 assets using raw SQL FOR UPDATE SKIP LOCKED and mark them as 'purging'
// We include a timeout check (1 hour) to allow picking up stalled tasks.
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string; key: string | null }[]>`
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string }[]>`
WITH to_purge AS (
SELECT id FROM assets
WHERE status = 'pending_purge'
@@ -1055,46 +1077,14 @@ export class AssetService {
)
UPDATE assets SET status = 'purging', updated_at = NOW()
WHERE id IN (SELECT id FROM to_purge)
RETURNING id, key;
RETURNING id;
`
if (lockedAssets.length === 0) return
let physicalFilesDeleted = 0
let physicalFoldersDeleted = 0
const successfulIds = lockedAssets.map((a) => a.id)
// 2. Physical file deletion from storage
const bucket = process.env.S3_BUCKET || 'shumai'
const results = await Promise.all(
lockedAssets.map(async (a) => {
if (a.key) {
try {
// Check if this asset has a dedicated directory (e.g., file/ULID/raw)
// If it has more than one slash, we delete the entire containing directory
// to ensure transcodes/thumbnails are also removed.
const parts = a.key.split('/')
if (parts.length > 2) {
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
const count = await s3Service.deletePrefix(bucket, prefix)
physicalFilesDeleted += count
physicalFoldersDeleted += 1
} else {
const count = await s3Service.deleteObject(bucket, a.key)
physicalFilesDeleted += count
}
return a.id
} catch (e: unknown) {
console.error(`Failed to delete storage object(s) for ${a.key}:`, e)
return null
}
}
// Assets without keys (folders, etc.) are considered processed
return a.id
}),
)
// 3. Hard delete from database ONLY for assets successfully removed from storage
const successfulIds = results.filter((id): id is string => id !== null)
// 3. Hard delete from database
if (successfulIds.length > 0) {
await this.prismaClient.asset.deleteMany({
where: { id: { in: successfulIds } },
@@ -1103,10 +1093,64 @@ export class AssetService {
logger.info(
{
purgedCount: successfulIds.length,
physicalFilesDeleted,
physicalFoldersDeleted,
},
`${successfulIds.length} assets purged, deleted ${physicalFilesDeleted} files in ${physicalFoldersDeleted} folders`,
`${successfulIds.length} assets purged from database`,
)
}
}
/**
* Stage 3: Garbage Collection - Physically delete files and StorageKey records
* that have no associated logical assets.
*/
private async purgeUnreferencedStorageKeys() {
const bucket = process.env.S3_BUCKET || 'shumai'
let purgedCount = 0
let physicalFilesDeleted = 0
// 1. Find storage keys with no associated assets and older than 24 hours
// We use 24 hours to avoid race conditions with ongoing uploads or copies.
const twentyFourHoursAgo = new Date()
twentyFourHoursAgo.setDate(twentyFourHoursAgo.getDate() - 1)
const unreferencedKeys = await this.prismaClient.storageKey.findMany({
where: {
assets: { none: {} },
createdAt: { lt: twentyFourHoursAgo },
},
take: 100,
})
for (const sk of unreferencedKeys) {
try {
// Physically delete from S3
const parts = sk.key.split('/')
if (parts.length > 2) {
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
const count = await s3Service.deletePrefix(bucket, prefix)
physicalFilesDeleted += count
} else {
const count = await s3Service.deleteObject(bucket, sk.key)
physicalFilesDeleted += count
}
// Delete from database
await this.prismaClient.storageKey.delete({
where: { id: sk.id },
})
purgedCount++
} catch (e: unknown) {
logger.error(
{ key: sk.key, error: e instanceof Error ? e.message : String(e) },
'Failed to purge unreferenced storage key',
)
}
}
if (purgedCount > 0) {
logger.info(
{ purgedCount, physicalFilesDeleted },
`Garbage collection: purged ${purgedCount} storage keys and ${physicalFilesDeleted} physical files`,
)
}
}
@@ -1280,12 +1324,12 @@ export class AssetService {
where: { id: commentId },
include: {
creator: true,
asset: true,
attachments: { include: { asset: true } },
asset: { include: { storageKey: true } },
attachments: { include: { asset: { include: { storageKey: true } } } },
replies: {
include: {
creator: true,
attachments: { include: { asset: true } },
attachments: { include: { asset: { include: { storageKey: true } } } },
},
orderBy: { id: 'asc' },
},
@@ -1307,12 +1351,12 @@ export class AssetService {
where: { assetId: resolvedAssetId, replyToId: null },
include: {
creator: true,
asset: true,
attachments: { include: { asset: true } },
asset: { include: { storageKey: true } },
attachments: { include: { asset: { include: { storageKey: true } } } },
replies: {
include: {
creator: true,
attachments: { include: { asset: true } },
attachments: { include: { asset: { include: { storageKey: true } } } },
},
orderBy: { id: 'asc' },
},
@@ -1363,7 +1407,7 @@ export class AssetService {
if (stackIds.size > 0) {
const allVersions = await this.prismaClient.asset.findMany({
where: { parentId: { in: Array.from(stackIds) }, isDeleted: false },
include: { creator: true, metadataValues: true },
include: { creator: true, metadataValues: true, storageKey: true },
})
for (const v of allVersions) {
const list = versionsMap.get(v.parentId!) || []
@@ -1482,6 +1526,16 @@ export class AssetService {
)
}
const key = latestVersion.storageKey?.key
if (media && key) {
media.original = {
key,
downloadUrl: await s3Service.presign(process.env.S3_BUCKET || 'shumai', key, 'GET'),
filesizeInBytes: latestVersion.sizeByte,
codec: '', // TODO: extract from metadata if needed
}
}
result.push({
id: a.id,
name:
@@ -1591,8 +1645,12 @@ export class AssetService {
const attachments: AttachmentInfo[] = []
if (c.attachments) {
for (const a of c.attachments) {
if (a.asset?.key) {
const url = await s3Service.presign(process.env.S3_BUCKET || 'shumai', a.asset.key, 'GET')
if (a.asset?.storageKey?.key) {
const url = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
a.asset.storageKey.key,
'GET',
)
attachments.push({
id: a.id,
assetId: a.asset.id,
+25 -5
View File
@@ -241,8 +241,13 @@ describe('ProjectService', () => {
name: 'Test Asset',
type: 'file',
status: 'processed',
projectId: project.id,
key: 'asset-key',
project: { connect: { id: project.id } },
storageKey: {
connectOrCreate: {
where: { key: 'asset-key' },
create: { key: 'asset-key' },
},
},
},
})
@@ -310,9 +315,14 @@ describe('ProjectService', () => {
name: 'File C',
type: 'file',
status: 'processed',
projectId: project.id,
parentId: folderB.id,
key: 'file-c-key',
project: { connect: { id: project.id } },
parent: { connect: { id: folderB.id } },
storageKey: {
connectOrCreate: {
where: { key: 'file-c-key' },
create: { key: 'file-c-key' },
},
},
},
})
@@ -337,9 +347,19 @@ describe('ProjectService', () => {
// Manually trigger the private cleanup methods for testing
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).expireTrashedAssets()
// Trigger the purge job
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgePendingAssets()
// Manually set storage key createdAt to the past so GC picks it up
await prisma.storageKey.updateMany({
data: { createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000) },
})
// Trigger GC job
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgeUnreferencedStorageKeys()
// Verify everything is wiped from DB
expect(await prisma.project.findUnique({ where: { id: project.id } })).toBeNull()
expect(await prisma.asset.findUnique({ where: { id: folderA.id } })).toBeNull()
+1 -1
View File
@@ -39,7 +39,7 @@ describe('SkillService', () => {
const asset = await prisma.asset.create({
data: {
name: 'test.zip',
key: 'test.zip',
storageKey: { create: { key: 'test.zip' } },
type: 'file',
status: 'uploaded',
},
+5 -3
View File
@@ -25,9 +25,11 @@ export class SkillService {
} else if (req.assetId) {
const asset = await this.prismaClient.asset.findUnique({
where: { id: req.assetId },
include: { storageKey: true },
})
if (!asset || !asset.key) throw new Error('Asset not found or has no key')
const { buffer } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', asset.key)
const key = asset?.storageKey?.key
if (!asset || !key) throw new Error('Asset not found or has no key')
const { buffer } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', key)
zipBuffer = buffer
} else {
throw new Error('Either assetId or githubUrl must be provided')
@@ -68,7 +70,7 @@ export class SkillService {
const skillAsset = await this.prismaClient.asset.create({
data: {
name: `${name}.zip`,
key: assetKey,
storageKey: { create: { key: assetKey } },
type: 'file',
mediaType: 'application/zip',
sizeByte: repackedZip.length,
+32 -12
View File
@@ -101,10 +101,15 @@ describe('UploadService', () => {
data: {
name: 'video.mp4',
type: AssetType.file,
projectId,
parentId,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
status: AssetStatus.uploading,
key: 'test-key',
storageKey: {
connectOrCreate: {
where: { key: 'test-key' },
create: { key: 'test-key' },
},
},
mediaType: 'video/mp4',
},
})
@@ -134,10 +139,15 @@ describe('UploadService', () => {
data: {
name: 'image.png',
type: AssetType.file,
projectId,
parentId,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
status: AssetStatus.uploading,
key: 'test-key',
storageKey: {
connectOrCreate: {
where: { key: 'test-key' },
create: { key: 'test-key' },
},
},
mediaType: 'image/png',
},
})
@@ -162,10 +172,15 @@ describe('UploadService', () => {
data: {
name: 'file.txt',
type: AssetType.file,
projectId,
parentId,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
status: AssetStatus.uploading,
key: 'test-key',
storageKey: {
connectOrCreate: {
where: { key: 'test-key' },
create: { key: 'test-key' },
},
},
mediaType: 'text/plain',
},
})
@@ -259,10 +274,15 @@ describe('UploadService', () => {
data: {
name: 'v2.txt',
type: AssetType.file,
projectId,
parentId: stack.id,
project: { connect: { id: projectId } },
parent: { connect: { id: stack.id } },
status: AssetStatus.uploading,
key: 'v2-key',
storageKey: {
connectOrCreate: {
where: { key: 'v2-key' },
create: { key: 'v2-key' },
},
},
sizeByte: 2000,
},
})
+12 -8
View File
@@ -127,15 +127,15 @@ export class UploadService {
data: {
name: file.name,
type: assetType,
key,
storageKey: key ? { create: { key } } : undefined,
sortIndex: newSortIndex,
mediaType: file.mediaType,
status: AssetStatus.uploading,
sizeByte: file.size,
creatorId: userId,
parentId,
projectId,
taskId,
creator: userId ? { connect: { id: userId } } : undefined,
parent: parentId ? { connect: { id: parentId } } : undefined,
project: { connect: { id: projectId } },
task: { connect: { id: taskId } },
},
})
createdIds.push(newAsset.id)
@@ -168,11 +168,15 @@ export class UploadService {
): Promise<void> {
const asset = await this.prismaClient.asset.findUnique({
where: { id: req.fileId },
include: { project: { include: { team: true } } },
include: {
storageKey: true,
project: { include: { team: true } },
},
})
if (!asset) throw new Error('Asset not found')
if (!asset.project) throw new Error('Project not found for asset')
if (!asset.key) throw new Error('Asset has no key')
const key = asset.storageKey?.key
if (!key) throw new Error('Asset has no key')
if (req.errorMessage) {
await this.prismaClient.asset.delete({ where: { id: asset.id } })
@@ -192,7 +196,7 @@ export class UploadService {
return
}
const size = await s3Service.getObjectSize(process.env.S3_BUCKET || 'shumai', asset.key)
const size = await s3Service.getObjectSize(process.env.S3_BUCKET || 'shumai', key)
const project = asset.project
+1 -1
View File
@@ -93,7 +93,7 @@ describe('Transcode Activities', () => {
const asset = await prisma.asset.create({
data: {
name: 'v.mp4',
key: 'v.mp4',
storageKey: { create: { key: 'v.mp4' } },
status: 'uploaded',
type: 'file',
},
+7 -3
View File
@@ -255,10 +255,14 @@ export interface UpdateAssetMediaActivityParams {
export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityParams) {
const bucket = process.env.S3_BUCKET || 'shumai'
const asset = await prisma.asset.findUnique({ where: { id: params.assetId } })
if (!asset || !asset.key) throw new Error('Asset not found')
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: { storageKey: true },
})
const key = asset?.storageKey?.key
if (!asset || !key) throw new Error('Asset not found or has no key')
const infoKey = path.join(path.dirname(asset.key), 'info.json')
const infoKey = path.join(path.dirname(key), 'info.json')
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
+1 -1
View File
@@ -90,7 +90,7 @@ describe('Transcode Workflow', () => {
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-1',
key: 'video.mp4',
storageKey: { key: 'video.mp4' },
mediaType: 'video/mp4',
})
+11 -10
View File
@@ -38,14 +38,15 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
// 2. Get Asset
const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId)
if (!asset || !asset.key) throw new Error('Asset not found')
const key = asset?.storageKey?.key
if (!asset || !key) throw new Error('Asset not found or has no key')
// 3. Get Worker-Specific Queue for Transcode
workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity)
// 4. Download Media to Tmp
const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, {
assetKey: asset.key,
assetKey: key,
})
const { filePath } = download
tmpDir = download.tmpDir
@@ -59,7 +60,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
})
mediaInfo.original = {
key: asset.key,
key,
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
@@ -89,7 +90,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
}
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
assetKey: asset.key,
assetKey: key,
filePath,
videoSpec,
duration: metadata.duration,
@@ -125,7 +126,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
if (imageSpec.width > metadata.originalWidth) continue
const imageTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: asset.key,
assetKey: key,
filePath,
imageSpec,
})
@@ -143,7 +144,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
format: 'webp',
}
const imageTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: asset.key,
assetKey: key,
filePath,
imageSpec,
})
@@ -164,7 +165,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
// 9. System: Thumbnail
if (spec.thumbnail) {
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: asset.key,
assetKey: key,
filePath,
imageSpec: { width: 480, height: 0, quality: 80, format: 'webp' },
})
@@ -173,8 +174,8 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
// 10. System: Sprite/Poster (Video only usually)
if (spec.sprite || spec.poster) {
const lastSlashIndex = asset.key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : asset.key.substring(0, lastSlashIndex)
const lastSlashIndex = key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
const spriteSpec: PrismaJson.SpriteInfo = {
key: assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp',
@@ -187,7 +188,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
}
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
assetKey: asset.key,
assetKey: key,
filePath,
spriteSpec,
posterSpec,
+3 -3
View File
@@ -25,10 +25,10 @@ describe('AI Activities', () => {
const asset = await prisma.asset.create({
data: {
name: 'test.png',
key: 'test.png',
storageKey: { create: { key: 'test.png' } },
status: 'uploaded',
type: 'file',
projectId: project.id,
project: { connect: { id: project.id } },
},
})
@@ -54,7 +54,7 @@ describe('AI Activities', () => {
const asset = await prisma.asset.create({
data: {
name: 'test.png',
key: 'test.png',
storageKey: { create: { key: 'test.png' } },
status: 'uploaded',
type: 'file',
},
+2 -1
View File
@@ -11,6 +11,7 @@ export async function getAssetActivity(assetId: string) {
return prisma.asset.findUnique({
where: { id: assetId },
include: {
storageKey: true,
project: {
include: {
team: true,
@@ -46,7 +47,7 @@ export async function updateAssetMetadataActivity(params: UpdateAssetMetadataPar
export async function getCommentActivity(commentId: string) {
return prisma.assetComment.findUnique({
where: { id: commentId },
include: { attachments: { include: { asset: true } } },
include: { attachments: { include: { asset: { include: { storageKey: true } } } } },
})
}
+61
View File
@@ -0,0 +1,61 @@
import { prisma } from '@/db'
import { s3Service } from '@/services/s3/s3'
import { logger } from '@/logger'
export async function purgeUnreferencedStorageKeysActivity(): Promise<{
purgedCount: number
physicalFilesDeleted: number
}> {
const bucket = process.env.S3_BUCKET || 'shumai'
let purgedCount = 0
let physicalFilesDeleted = 0
// 1. Find storage keys with no associated assets and older than 24 hours
// We use 24 hours to avoid race conditions with ongoing uploads or copies.
const twentyFourHoursAgo = new Date()
twentyFourHoursAgo.setDate(twentyFourHoursAgo.getDate() - 1)
const unreferencedKeys = await prisma.storageKey.findMany({
where: {
assets: { none: {} },
createdAt: { lt: twentyFourHoursAgo },
},
take: 100,
})
for (const sk of unreferencedKeys) {
try {
// Physically delete from S3
// Handle complex keys (directories) same as before
const parts = sk.key.split('/')
if (parts.length > 2) {
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
const count = await s3Service.deletePrefix(bucket, prefix)
physicalFilesDeleted += count
} else {
const count = await s3Service.deleteObject(bucket, sk.key)
physicalFilesDeleted += count
}
// Delete from database
await prisma.storageKey.delete({
where: { id: sk.id },
})
purgedCount++
} catch (e: unknown) {
logger.error(
{ key: sk.key, error: e instanceof Error ? e.message : String(e) },
'Failed to purge unreferenced storage key',
)
}
}
if (purgedCount > 0) {
logger.info(
{ purgedCount, physicalFilesDeleted },
`Garbage collection: purged ${purgedCount} storage keys and ${physicalFilesDeleted} physical files`,
)
}
return { purgedCount, physicalFilesDeleted }
}
+2
View File
@@ -3,6 +3,7 @@ import * as aiActivities from './ai'
import * as transcodeActivities from '@/transcode/activities/transcode'
import * as agentActivities from './agent'
import * as dbActivities from './db'
import * as gcActivities from './gc'
export const activities = {
...taskActivities,
@@ -10,4 +11,5 @@ export const activities = {
...transcodeActivities,
...agentActivities,
...dbActivities,
...gcActivities,
}
+1 -1
View File
@@ -33,7 +33,7 @@ describe('Task Activities', () => {
const asset = await prisma.asset.create({
data: {
name: 'test.mp4',
key: 'test.mp4',
storageKey: { create: { key: 'test.mp4' } },
status: 'uploaded',
type: 'file',
},
+4 -4
View File
@@ -56,10 +56,10 @@ describe('WorkflowService', () => {
const asset = await prisma.asset.create({
data: {
name: 'test.mp3',
key: 'test/test.mp3',
storageKey: { create: { key: 'test/test.mp3' } },
status: 'uploaded',
type: 'file',
projectId: project.id,
project: { connect: { id: project.id } },
},
})
@@ -100,11 +100,11 @@ describe('WorkflowService', () => {
const asset = await prisma.asset.create({
data: {
name: 'test.png',
key: 'test/test.png',
storageKey: { create: { key: 'test/test.png' } },
status: 'uploaded',
type: 'file',
mediaType: 'image/png',
projectId: project.id,
project: { connect: { id: project.id } },
},
})
+2 -2
View File
@@ -89,7 +89,7 @@ describe('AIAutofill Workflow', () => {
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-1',
key: 'test.mp4',
storageKey: { key: 'test.mp4' },
mediaType: 'video/mp4',
project: { id: 'proj-1', teamId: 'team-1' },
})
@@ -140,7 +140,7 @@ describe('AIAutofill Workflow', () => {
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-1',
key: 'test.mp4',
storageKey: { key: 'test.mp4' },
mediaType: 'video/mp4',
project: { id: 'proj-1', teamId: 'team-1' },
})
+4 -3
View File
@@ -47,7 +47,8 @@ export async function aiAutofillMedia(task: WorkflowTask): Promise<void> {
// 1. Get Asset
const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId)
if (!asset || !asset.project) {
const key = asset?.storageKey?.key
if (!asset || !asset.project || !key) {
throw new Error('Asset or project not found')
}
const teamId = asset.project.teamId
@@ -63,13 +64,13 @@ export async function aiAutofillMedia(task: WorkflowTask): Promise<void> {
)
const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, {
assetKey: asset.key || '',
assetKey: key,
})
const { filePath } = download
tmpDir = download.tmpDir
const generatedFiles = await executeActivity(transcodeWorkerQueue, extractAiMetadataActivity, {
assetKey: asset.key || '',
assetKey: key,
filePath,
type: 'autofill',
isImage,
+4 -3
View File
@@ -42,7 +42,8 @@ export async function aiTranscriptionMedia(task: WorkflowTask): Promise<void> {
// 1. Get Asset
const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId)
if (!asset || !asset.key) {
const key = asset?.storageKey?.key
if (!asset || !key) {
throw new Error('Asset not found')
}
@@ -53,13 +54,13 @@ export async function aiTranscriptionMedia(task: WorkflowTask): Promise<void> {
)
const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, {
assetKey: asset.key,
assetKey: key,
})
const { filePath } = download
tmpDir = download.tmpDir
const generatedFiles = await executeActivity(transcodeWorkerQueue, extractAiMetadataActivity, {
assetKey: asset.key,
assetKey: key,
filePath,
type: 'transcription',
isImage: false,