mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 03:59:29 +00:00
feat(asset): decouple logical assets from physical storage with StorageKey and implement O(1) copy
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `key` on the `assets` table. All the data in the column will be lost.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "assets" DROP COLUMN "key",
|
||||
ADD COLUMN "storage_key_id" TEXT;
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "storage_keys" (
|
||||
"id" TEXT NOT NULL,
|
||||
"key" TEXT NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "storage_keys_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "storage_keys_key_key" ON "storage_keys"("key");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "assets" ADD CONSTRAINT "assets_storage_key_id_fkey" FOREIGN KEY ("storage_key_id") REFERENCES "storage_keys"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
+13
-1
@@ -347,7 +347,6 @@ model Asset {
|
||||
id String @id @default(ulid())
|
||||
name String @default("")
|
||||
nameNgram String[] @default([]) @map("name_ngram")
|
||||
key String?
|
||||
type AssetType
|
||||
mediaType String? @map("media_type")
|
||||
fileCount Int @default(0) @map("file_count")
|
||||
@@ -370,6 +369,9 @@ model Asset {
|
||||
target Asset? @relation("SymlinkTarget", fields: [targetId], references: [id])
|
||||
symlinks Asset[] @relation("SymlinkTarget")
|
||||
|
||||
storageKeyId String? @map("storage_key_id")
|
||||
storageKey StorageKey? @relation(fields: [storageKeyId], references: [id])
|
||||
|
||||
metadataValues AssetMetadataValue[] @relation("AssetToMetadataValues")
|
||||
|
||||
creatorId String? @map("creator_id")
|
||||
@@ -403,6 +405,16 @@ model Asset {
|
||||
@@unique([parentId, targetId], name: "parentIdTargetId")
|
||||
}
|
||||
|
||||
model StorageKey {
|
||||
id String @id @default(ulid())
|
||||
key String @unique
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
assets Asset[]
|
||||
|
||||
@@map("storage_keys")
|
||||
}
|
||||
|
||||
model AssetMetadataValue {
|
||||
id String @id @default(ulid())
|
||||
assetId String @map("asset_id")
|
||||
|
||||
@@ -76,7 +76,7 @@ describe('readSkillTool', () => {
|
||||
data: {
|
||||
id: 'asset1',
|
||||
name: 'skill.zip',
|
||||
key: 'skills/skill.zip',
|
||||
storageKey: { create: { key: 'skills/skill.zip' } },
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
},
|
||||
|
||||
@@ -65,15 +65,16 @@ export const createReadSkillTool = (
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: skill.assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset || !asset.key) {
|
||||
if (!asset || !asset.storageKey?.key) {
|
||||
throw new Error('Skill asset not found or has no key')
|
||||
}
|
||||
|
||||
const { buffer: zipBuffer } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
asset.key,
|
||||
asset.storageKey.key,
|
||||
)
|
||||
|
||||
const zip = new AdmZip(zipBuffer)
|
||||
|
||||
Generated
+5
@@ -82,6 +82,11 @@ export type ShareLink = Prisma.ShareLinkModel
|
||||
*
|
||||
*/
|
||||
export type Asset = Prisma.AssetModel
|
||||
/**
|
||||
* Model StorageKey
|
||||
*
|
||||
*/
|
||||
export type StorageKey = Prisma.StorageKeyModel
|
||||
/**
|
||||
* Model AssetMetadataValue
|
||||
*
|
||||
|
||||
Generated
+5
@@ -106,6 +106,11 @@ export type ShareLink = Prisma.ShareLinkModel
|
||||
*
|
||||
*/
|
||||
export type Asset = Prisma.AssetModel
|
||||
/**
|
||||
* Model StorageKey
|
||||
*
|
||||
*/
|
||||
export type StorageKey = Prisma.StorageKeyModel
|
||||
/**
|
||||
* Model AssetMetadataValue
|
||||
*
|
||||
|
||||
Generated
+14
-4
File diff suppressed because one or more lines are too long
+88
-2
@@ -397,6 +397,7 @@ export const ModelName = {
|
||||
ProjectMember: 'ProjectMember',
|
||||
ShareLink: 'ShareLink',
|
||||
Asset: 'Asset',
|
||||
StorageKey: 'StorageKey',
|
||||
AssetMetadataValue: 'AssetMetadataValue',
|
||||
AssetComment: 'AssetComment',
|
||||
AssetCommentAttachment: 'AssetCommentAttachment',
|
||||
@@ -429,7 +430,7 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
|
||||
omit: GlobalOmitOptions
|
||||
}
|
||||
meta: {
|
||||
modelProps: "user" | "session" | "account" | "verification" | "authToken" | "team" | "sandbox" | "skill" | "teamMember" | "project" | "projectMember" | "shareLink" | "asset" | "assetMetadataValue" | "assetComment" | "assetCommentAttachment" | "assetEmbedding" | "invite" | "metadataField" | "notification" | "systemSettings" | "task" | "workflowTask" | "agent" | "agentSkill" | "agentSession" | "agentSessionEntry" | "userMetadata" | "model" | "provider"
|
||||
modelProps: "user" | "session" | "account" | "verification" | "authToken" | "team" | "sandbox" | "skill" | "teamMember" | "project" | "projectMember" | "shareLink" | "asset" | "storageKey" | "assetMetadataValue" | "assetComment" | "assetCommentAttachment" | "assetEmbedding" | "invite" | "metadataField" | "notification" | "systemSettings" | "task" | "workflowTask" | "agent" | "agentSkill" | "agentSession" | "agentSessionEntry" | "userMetadata" | "model" | "provider"
|
||||
txIsolationLevel: TransactionIsolationLevel
|
||||
}
|
||||
model: {
|
||||
@@ -1395,6 +1396,80 @@ export type TypeMap<ExtArgs extends runtime.Types.Extensions.InternalArgs = runt
|
||||
}
|
||||
}
|
||||
}
|
||||
StorageKey: {
|
||||
payload: Prisma.$StorageKeyPayload<ExtArgs>
|
||||
fields: Prisma.StorageKeyFieldRefs
|
||||
operations: {
|
||||
findUnique: {
|
||||
args: Prisma.StorageKeyFindUniqueArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload> | null
|
||||
}
|
||||
findUniqueOrThrow: {
|
||||
args: Prisma.StorageKeyFindUniqueOrThrowArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
|
||||
}
|
||||
findFirst: {
|
||||
args: Prisma.StorageKeyFindFirstArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload> | null
|
||||
}
|
||||
findFirstOrThrow: {
|
||||
args: Prisma.StorageKeyFindFirstOrThrowArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
|
||||
}
|
||||
findMany: {
|
||||
args: Prisma.StorageKeyFindManyArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>[]
|
||||
}
|
||||
create: {
|
||||
args: Prisma.StorageKeyCreateArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
|
||||
}
|
||||
createMany: {
|
||||
args: Prisma.StorageKeyCreateManyArgs<ExtArgs>
|
||||
result: BatchPayload
|
||||
}
|
||||
createManyAndReturn: {
|
||||
args: Prisma.StorageKeyCreateManyAndReturnArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>[]
|
||||
}
|
||||
delete: {
|
||||
args: Prisma.StorageKeyDeleteArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
|
||||
}
|
||||
update: {
|
||||
args: Prisma.StorageKeyUpdateArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
|
||||
}
|
||||
deleteMany: {
|
||||
args: Prisma.StorageKeyDeleteManyArgs<ExtArgs>
|
||||
result: BatchPayload
|
||||
}
|
||||
updateMany: {
|
||||
args: Prisma.StorageKeyUpdateManyArgs<ExtArgs>
|
||||
result: BatchPayload
|
||||
}
|
||||
updateManyAndReturn: {
|
||||
args: Prisma.StorageKeyUpdateManyAndReturnArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>[]
|
||||
}
|
||||
upsert: {
|
||||
args: Prisma.StorageKeyUpsertArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.PayloadToResult<Prisma.$StorageKeyPayload>
|
||||
}
|
||||
aggregate: {
|
||||
args: Prisma.StorageKeyAggregateArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.Optional<Prisma.AggregateStorageKey>
|
||||
}
|
||||
groupBy: {
|
||||
args: Prisma.StorageKeyGroupByArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.Optional<Prisma.StorageKeyGroupByOutputType>[]
|
||||
}
|
||||
count: {
|
||||
args: Prisma.StorageKeyCountArgs<ExtArgs>
|
||||
result: runtime.Types.Utils.Optional<Prisma.StorageKeyCountAggregateOutputType> | number
|
||||
}
|
||||
}
|
||||
}
|
||||
AssetMetadataValue: {
|
||||
payload: Prisma.$AssetMetadataValuePayload<ExtArgs>
|
||||
fields: Prisma.AssetMetadataValueFieldRefs
|
||||
@@ -2863,7 +2938,6 @@ export const AssetScalarFieldEnum = {
|
||||
id: 'id',
|
||||
name: 'name',
|
||||
nameNgram: 'nameNgram',
|
||||
key: 'key',
|
||||
type: 'type',
|
||||
mediaType: 'mediaType',
|
||||
fileCount: 'fileCount',
|
||||
@@ -2878,6 +2952,7 @@ export const AssetScalarFieldEnum = {
|
||||
updatedAt: 'updatedAt',
|
||||
parentId: 'parentId',
|
||||
targetId: 'targetId',
|
||||
storageKeyId: 'storageKeyId',
|
||||
creatorId: 'creatorId',
|
||||
taskId: 'taskId',
|
||||
projectId: 'projectId'
|
||||
@@ -2886,6 +2961,16 @@ export const AssetScalarFieldEnum = {
|
||||
export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof AssetScalarFieldEnum]
|
||||
|
||||
|
||||
export const StorageKeyScalarFieldEnum = {
|
||||
id: 'id',
|
||||
key: 'key',
|
||||
createdAt: 'createdAt',
|
||||
updatedAt: 'updatedAt'
|
||||
} as const
|
||||
|
||||
export type StorageKeyScalarFieldEnum = (typeof StorageKeyScalarFieldEnum)[keyof typeof StorageKeyScalarFieldEnum]
|
||||
|
||||
|
||||
export const AssetMetadataValueScalarFieldEnum = {
|
||||
id: 'id',
|
||||
assetId: 'assetId',
|
||||
@@ -3554,6 +3639,7 @@ export type GlobalOmitConfig = {
|
||||
projectMember?: Prisma.ProjectMemberOmit
|
||||
shareLink?: Prisma.ShareLinkOmit
|
||||
asset?: Prisma.AssetOmit
|
||||
storageKey?: Prisma.StorageKeyOmit
|
||||
assetMetadataValue?: Prisma.AssetMetadataValueOmit
|
||||
assetComment?: Prisma.AssetCommentOmit
|
||||
assetCommentAttachment?: Prisma.AssetCommentAttachmentOmit
|
||||
|
||||
+12
-1
@@ -64,6 +64,7 @@ export const ModelName = {
|
||||
ProjectMember: 'ProjectMember',
|
||||
ShareLink: 'ShareLink',
|
||||
Asset: 'Asset',
|
||||
StorageKey: 'StorageKey',
|
||||
AssetMetadataValue: 'AssetMetadataValue',
|
||||
AssetComment: 'AssetComment',
|
||||
AssetCommentAttachment: 'AssetCommentAttachment',
|
||||
@@ -270,7 +271,6 @@ export const AssetScalarFieldEnum = {
|
||||
id: 'id',
|
||||
name: 'name',
|
||||
nameNgram: 'nameNgram',
|
||||
key: 'key',
|
||||
type: 'type',
|
||||
mediaType: 'mediaType',
|
||||
fileCount: 'fileCount',
|
||||
@@ -285,6 +285,7 @@ export const AssetScalarFieldEnum = {
|
||||
updatedAt: 'updatedAt',
|
||||
parentId: 'parentId',
|
||||
targetId: 'targetId',
|
||||
storageKeyId: 'storageKeyId',
|
||||
creatorId: 'creatorId',
|
||||
taskId: 'taskId',
|
||||
projectId: 'projectId'
|
||||
@@ -293,6 +294,16 @@ export const AssetScalarFieldEnum = {
|
||||
export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof AssetScalarFieldEnum]
|
||||
|
||||
|
||||
export const StorageKeyScalarFieldEnum = {
|
||||
id: 'id',
|
||||
key: 'key',
|
||||
createdAt: 'createdAt',
|
||||
updatedAt: 'updatedAt'
|
||||
} as const
|
||||
|
||||
export type StorageKeyScalarFieldEnum = (typeof StorageKeyScalarFieldEnum)[keyof typeof StorageKeyScalarFieldEnum]
|
||||
|
||||
|
||||
export const AssetMetadataValueScalarFieldEnum = {
|
||||
id: 'id',
|
||||
assetId: 'assetId',
|
||||
|
||||
Generated
+1
@@ -21,6 +21,7 @@ export type * from './models/Project.ts'
|
||||
export type * from './models/ProjectMember.ts'
|
||||
export type * from './models/ShareLink.ts'
|
||||
export type * from './models/Asset.ts'
|
||||
export type * from './models/StorageKey.ts'
|
||||
export type * from './models/AssetMetadataValue.ts'
|
||||
export type * from './models/AssetComment.ts'
|
||||
export type * from './models/AssetCommentAttachment.ts'
|
||||
|
||||
Generated
+386
-107
File diff suppressed because it is too large
Load Diff
+1299
File diff suppressed because it is too large
Load Diff
@@ -118,6 +118,7 @@ export class AiService {
|
||||
|
||||
const asset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) throw new Error('failed to get asset')
|
||||
@@ -146,11 +147,12 @@ export class AiService {
|
||||
throw new Error(`unsupported media type for embeddings: ${asset.mediaType}`)
|
||||
}
|
||||
|
||||
if (!asset.key) {
|
||||
const key = asset.storageKey?.key
|
||||
if (!key) {
|
||||
throw new Error(`asset has no key`)
|
||||
}
|
||||
|
||||
const { buffer: data } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', asset.key)
|
||||
const { buffer: data } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', key)
|
||||
|
||||
if (isImage) {
|
||||
const embVec = await p.generateImageEmbedding(config.model, data)
|
||||
|
||||
@@ -1293,25 +1293,39 @@ describe('AssetService', () => {
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.pending_purge,
|
||||
isDeleted: true,
|
||||
projectId: project.id,
|
||||
parentId: folderA.id,
|
||||
key: `key-${i}`,
|
||||
project: { connect: { id: project.id } },
|
||||
parent: { connect: { id: folderA.id } },
|
||||
storageKey: {
|
||||
create: { key: `key-${i}` },
|
||||
},
|
||||
}))
|
||||
|
||||
await prisma.asset.createMany({ data: childData })
|
||||
for (const data of childData) {
|
||||
const asset = await prisma.asset.create({
|
||||
data: data as any,
|
||||
include: { storageKey: true },
|
||||
})
|
||||
await prisma.storageKey.update({
|
||||
where: { id: asset.storageKeyId! },
|
||||
data: { createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000) },
|
||||
})
|
||||
}
|
||||
|
||||
// 3. Trigger the purge job for the FIRST batch (100 items)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgePendingAssets()
|
||||
|
||||
// 4. Verify some items were processed in S3
|
||||
// We don't know the order, but if Folder A was in the first 100,
|
||||
// then its synchronous cascade will have wiped the remaining children.
|
||||
|
||||
// 5. Trigger the purge job for the SECOND batch
|
||||
// ...
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgePendingAssets()
|
||||
|
||||
// 6. Trigger GC job to physically delete files
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgeUnreferencedStorageKeys()
|
||||
// Call it again to process the remaining items (batch size is 100)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgeUnreferencedStorageKeys()
|
||||
|
||||
// 6. Verify S3 deletion was called for EVERY child
|
||||
for (let i = 0; i < childCount; i++) {
|
||||
expect(s3Service.deleteObject).toHaveBeenCalledWith(expect.any(String), `key-${i}`)
|
||||
@@ -1334,24 +1348,29 @@ describe('AssetService', () => {
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.pending_purge,
|
||||
isDeleted: true,
|
||||
projectId: project.id,
|
||||
key: 'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
|
||||
project: { connect: { id: project.id } },
|
||||
storageKey: {
|
||||
create: {
|
||||
key: 'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
|
||||
createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// Trigger the purge job
|
||||
// 1. Purge the asset record
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgePendingAssets()
|
||||
|
||||
// 2. Trigger GC job
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgeUnreferencedStorageKeys()
|
||||
|
||||
// Verify deletePrefix was called with the correct prefix instead of just deleteObject
|
||||
expect(s3Service.deletePrefix).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
'file/01KSBRJVY3DPK111S2MECFKDQ4/',
|
||||
)
|
||||
expect(s3Service.deleteObject).not.toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
|
||||
)
|
||||
|
||||
// Verify record is gone
|
||||
expect(await prisma.asset.findUnique({ where: { id: complexFile.id } })).toBeNull()
|
||||
|
||||
+134
-76
@@ -18,7 +18,7 @@ import {
|
||||
UpdateAssetOrderRequest,
|
||||
UserInfo,
|
||||
} from '@/dtos/asset'
|
||||
import { Asset, AssetType, Prisma } from '@/generated/prisma/client.ts'
|
||||
import { Asset, AssetType, Prisma, StorageKey } from '@/generated/prisma/client.ts'
|
||||
import { PaginatedData, paginateQuery, PaginationParams } from '@/services/pagination'
|
||||
import { s3Service } from '@/services/s3/s3'
|
||||
import { generateKeyBetween } from 'jittered-fractional-indexing'
|
||||
@@ -29,17 +29,19 @@ type AssetWithIncludes = Prisma.AssetGetPayload<{
|
||||
include: {
|
||||
creator: true
|
||||
metadataValues: true
|
||||
storageKey: true
|
||||
target: {
|
||||
include: {
|
||||
creator: true
|
||||
metadataValues: true
|
||||
storageKey: true
|
||||
children: {
|
||||
include: { creator: true; metadataValues: true }
|
||||
include: { creator: true; metadataValues: true; storageKey: true }
|
||||
}
|
||||
}
|
||||
}
|
||||
children: {
|
||||
include: { creator: true; metadataValues: true }
|
||||
include: { creator: true; metadataValues: true; storageKey: true }
|
||||
}
|
||||
}
|
||||
}>
|
||||
@@ -47,11 +49,11 @@ type AssetWithIncludes = Prisma.AssetGetPayload<{
|
||||
type CommentWithIncludes = Prisma.AssetCommentGetPayload<{
|
||||
include: {
|
||||
creator: true
|
||||
attachments: { include: { asset: true } }
|
||||
attachments: { include: { asset: { include: { storageKey: true } } } }
|
||||
replies: {
|
||||
include: {
|
||||
creator: true
|
||||
attachments: { include: { asset: true } }
|
||||
attachments: { include: { asset: { include: { storageKey: true } } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -91,19 +93,21 @@ export class AssetService {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
@@ -335,27 +339,17 @@ export class AssetService {
|
||||
|
||||
private async copyAssetRecursive(
|
||||
tx: Prisma.TransactionClient,
|
||||
asset: Asset,
|
||||
asset: Asset & { storageKey?: StorageKey | null },
|
||||
newParentId: string | null,
|
||||
projectId: string | null,
|
||||
sortIndex: string | null,
|
||||
creatorId?: string,
|
||||
withComments: boolean = false,
|
||||
): Promise<Asset> {
|
||||
let newKey = asset.key
|
||||
if (asset.type === AssetType.file && asset.key) {
|
||||
// Duplicate S3 object
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const ext = path.extname(asset.key)
|
||||
newKey = ulid() + ext
|
||||
await s3Service.copyObject(bucket, asset.key, bucket, newKey)
|
||||
}
|
||||
|
||||
const newAsset = await tx.asset.create({
|
||||
data: {
|
||||
name: asset.name,
|
||||
nameNgram: asset.nameNgram,
|
||||
key: newKey,
|
||||
type: asset.type,
|
||||
mediaType: asset.mediaType,
|
||||
fileCount: asset.fileCount,
|
||||
@@ -372,6 +366,7 @@ export class AssetService {
|
||||
taskId: asset.taskId,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
targetId: asset.targetId as any,
|
||||
storageKeyId: asset.storageKeyId,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -635,19 +630,21 @@ export class AssetService {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
@@ -699,7 +696,14 @@ export class AssetService {
|
||||
|
||||
if (parent) data.parent = { connect: { id: parent.id } }
|
||||
if (sortIndex) data.sortIndex = sortIndex
|
||||
if (req.key) data.key = req.key
|
||||
if (req.key) {
|
||||
;(data as any).storageKey = {
|
||||
connectOrCreate: {
|
||||
where: { key: req.key },
|
||||
create: { key: req.key },
|
||||
},
|
||||
}
|
||||
}
|
||||
if (req.sizeByte) data.sizeByte = req.sizeByte
|
||||
if (req.contentType) data.mediaType = req.contentType
|
||||
if (req.creatorId) data.creator = { connect: { id: req.creatorId } }
|
||||
@@ -709,19 +713,21 @@ export class AssetService {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
@@ -746,19 +752,21 @@ export class AssetService {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
@@ -868,21 +876,23 @@ export class AssetService {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
storageKey: true,
|
||||
},
|
||||
})
|
||||
const infos = await this.toAssetInfos([asset])
|
||||
const infos = await this.toAssetInfos([asset as any])
|
||||
return infos[0]
|
||||
}
|
||||
|
||||
@@ -995,8 +1005,20 @@ export class AssetService {
|
||||
setTimeout(runPurge, 1000) // Run every 1 second
|
||||
}
|
||||
|
||||
// Stage 3: Garbage Collection Job - Purge unreferenced StorageKeys and physical files
|
||||
const runGc = async () => {
|
||||
if (!this.cleanupJobsRunning) return
|
||||
try {
|
||||
await this.purgeUnreferencedStorageKeys()
|
||||
} catch (e: unknown) {
|
||||
console.error('Error in asset GC job:', e)
|
||||
}
|
||||
setTimeout(runGc, 10000) // Run every 10 seconds
|
||||
}
|
||||
|
||||
runExpiration()
|
||||
runPurge()
|
||||
runGc()
|
||||
}
|
||||
|
||||
stopCleanupJob() {
|
||||
@@ -1045,7 +1067,7 @@ export class AssetService {
|
||||
private async purgePendingAssets() {
|
||||
// 1. Lock 100 assets using raw SQL FOR UPDATE SKIP LOCKED and mark them as 'purging'
|
||||
// We include a timeout check (1 hour) to allow picking up stalled tasks.
|
||||
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string; key: string | null }[]>`
|
||||
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string }[]>`
|
||||
WITH to_purge AS (
|
||||
SELECT id FROM assets
|
||||
WHERE status = 'pending_purge'
|
||||
@@ -1055,46 +1077,14 @@ export class AssetService {
|
||||
)
|
||||
UPDATE assets SET status = 'purging', updated_at = NOW()
|
||||
WHERE id IN (SELECT id FROM to_purge)
|
||||
RETURNING id, key;
|
||||
RETURNING id;
|
||||
`
|
||||
|
||||
if (lockedAssets.length === 0) return
|
||||
|
||||
let physicalFilesDeleted = 0
|
||||
let physicalFoldersDeleted = 0
|
||||
const successfulIds = lockedAssets.map((a) => a.id)
|
||||
|
||||
// 2. Physical file deletion from storage
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const results = await Promise.all(
|
||||
lockedAssets.map(async (a) => {
|
||||
if (a.key) {
|
||||
try {
|
||||
// Check if this asset has a dedicated directory (e.g., file/ULID/raw)
|
||||
// If it has more than one slash, we delete the entire containing directory
|
||||
// to ensure transcodes/thumbnails are also removed.
|
||||
const parts = a.key.split('/')
|
||||
if (parts.length > 2) {
|
||||
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
|
||||
const count = await s3Service.deletePrefix(bucket, prefix)
|
||||
physicalFilesDeleted += count
|
||||
physicalFoldersDeleted += 1
|
||||
} else {
|
||||
const count = await s3Service.deleteObject(bucket, a.key)
|
||||
physicalFilesDeleted += count
|
||||
}
|
||||
return a.id
|
||||
} catch (e: unknown) {
|
||||
console.error(`Failed to delete storage object(s) for ${a.key}:`, e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
// Assets without keys (folders, etc.) are considered processed
|
||||
return a.id
|
||||
}),
|
||||
)
|
||||
|
||||
// 3. Hard delete from database ONLY for assets successfully removed from storage
|
||||
const successfulIds = results.filter((id): id is string => id !== null)
|
||||
// 3. Hard delete from database
|
||||
if (successfulIds.length > 0) {
|
||||
await this.prismaClient.asset.deleteMany({
|
||||
where: { id: { in: successfulIds } },
|
||||
@@ -1103,10 +1093,64 @@ export class AssetService {
|
||||
logger.info(
|
||||
{
|
||||
purgedCount: successfulIds.length,
|
||||
physicalFilesDeleted,
|
||||
physicalFoldersDeleted,
|
||||
},
|
||||
`${successfulIds.length} assets purged, deleted ${physicalFilesDeleted} files in ${physicalFoldersDeleted} folders`,
|
||||
`${successfulIds.length} assets purged from database`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 3: Garbage Collection - Physically delete files and StorageKey records
|
||||
* that have no associated logical assets.
|
||||
*/
|
||||
private async purgeUnreferencedStorageKeys() {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
let purgedCount = 0
|
||||
let physicalFilesDeleted = 0
|
||||
|
||||
// 1. Find storage keys with no associated assets and older than 24 hours
|
||||
// We use 24 hours to avoid race conditions with ongoing uploads or copies.
|
||||
const twentyFourHoursAgo = new Date()
|
||||
twentyFourHoursAgo.setDate(twentyFourHoursAgo.getDate() - 1)
|
||||
|
||||
const unreferencedKeys = await this.prismaClient.storageKey.findMany({
|
||||
where: {
|
||||
assets: { none: {} },
|
||||
createdAt: { lt: twentyFourHoursAgo },
|
||||
},
|
||||
take: 100,
|
||||
})
|
||||
|
||||
for (const sk of unreferencedKeys) {
|
||||
try {
|
||||
// Physically delete from S3
|
||||
const parts = sk.key.split('/')
|
||||
if (parts.length > 2) {
|
||||
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
|
||||
const count = await s3Service.deletePrefix(bucket, prefix)
|
||||
physicalFilesDeleted += count
|
||||
} else {
|
||||
const count = await s3Service.deleteObject(bucket, sk.key)
|
||||
physicalFilesDeleted += count
|
||||
}
|
||||
|
||||
// Delete from database
|
||||
await this.prismaClient.storageKey.delete({
|
||||
where: { id: sk.id },
|
||||
})
|
||||
purgedCount++
|
||||
} catch (e: unknown) {
|
||||
logger.error(
|
||||
{ key: sk.key, error: e instanceof Error ? e.message : String(e) },
|
||||
'Failed to purge unreferenced storage key',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (purgedCount > 0) {
|
||||
logger.info(
|
||||
{ purgedCount, physicalFilesDeleted },
|
||||
`Garbage collection: purged ${purgedCount} storage keys and ${physicalFilesDeleted} physical files`,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1280,12 +1324,12 @@ export class AssetService {
|
||||
where: { id: commentId },
|
||||
include: {
|
||||
creator: true,
|
||||
asset: true,
|
||||
attachments: { include: { asset: true } },
|
||||
asset: { include: { storageKey: true } },
|
||||
attachments: { include: { asset: { include: { storageKey: true } } } },
|
||||
replies: {
|
||||
include: {
|
||||
creator: true,
|
||||
attachments: { include: { asset: true } },
|
||||
attachments: { include: { asset: { include: { storageKey: true } } } },
|
||||
},
|
||||
orderBy: { id: 'asc' },
|
||||
},
|
||||
@@ -1307,12 +1351,12 @@ export class AssetService {
|
||||
where: { assetId: resolvedAssetId, replyToId: null },
|
||||
include: {
|
||||
creator: true,
|
||||
asset: true,
|
||||
attachments: { include: { asset: true } },
|
||||
asset: { include: { storageKey: true } },
|
||||
attachments: { include: { asset: { include: { storageKey: true } } } },
|
||||
replies: {
|
||||
include: {
|
||||
creator: true,
|
||||
attachments: { include: { asset: true } },
|
||||
attachments: { include: { asset: { include: { storageKey: true } } } },
|
||||
},
|
||||
orderBy: { id: 'asc' },
|
||||
},
|
||||
@@ -1363,7 +1407,7 @@ export class AssetService {
|
||||
if (stackIds.size > 0) {
|
||||
const allVersions = await this.prismaClient.asset.findMany({
|
||||
where: { parentId: { in: Array.from(stackIds) }, isDeleted: false },
|
||||
include: { creator: true, metadataValues: true },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
})
|
||||
for (const v of allVersions) {
|
||||
const list = versionsMap.get(v.parentId!) || []
|
||||
@@ -1482,6 +1526,16 @@ export class AssetService {
|
||||
)
|
||||
}
|
||||
|
||||
const key = latestVersion.storageKey?.key
|
||||
if (media && key) {
|
||||
media.original = {
|
||||
key,
|
||||
downloadUrl: await s3Service.presign(process.env.S3_BUCKET || 'shumai', key, 'GET'),
|
||||
filesizeInBytes: latestVersion.sizeByte,
|
||||
codec: '', // TODO: extract from metadata if needed
|
||||
}
|
||||
}
|
||||
|
||||
result.push({
|
||||
id: a.id,
|
||||
name:
|
||||
@@ -1591,8 +1645,12 @@ export class AssetService {
|
||||
const attachments: AttachmentInfo[] = []
|
||||
if (c.attachments) {
|
||||
for (const a of c.attachments) {
|
||||
if (a.asset?.key) {
|
||||
const url = await s3Service.presign(process.env.S3_BUCKET || 'shumai', a.asset.key, 'GET')
|
||||
if (a.asset?.storageKey?.key) {
|
||||
const url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
a.asset.storageKey.key,
|
||||
'GET',
|
||||
)
|
||||
attachments.push({
|
||||
id: a.id,
|
||||
assetId: a.asset.id,
|
||||
|
||||
@@ -241,8 +241,13 @@ describe('ProjectService', () => {
|
||||
name: 'Test Asset',
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
key: 'asset-key',
|
||||
project: { connect: { id: project.id } },
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'asset-key' },
|
||||
create: { key: 'asset-key' },
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -310,9 +315,14 @@ describe('ProjectService', () => {
|
||||
name: 'File C',
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
parentId: folderB.id,
|
||||
key: 'file-c-key',
|
||||
project: { connect: { id: project.id } },
|
||||
parent: { connect: { id: folderB.id } },
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'file-c-key' },
|
||||
create: { key: 'file-c-key' },
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -337,9 +347,19 @@ describe('ProjectService', () => {
|
||||
// Manually trigger the private cleanup methods for testing
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).expireTrashedAssets()
|
||||
// Trigger the purge job
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgePendingAssets()
|
||||
|
||||
// Manually set storage key createdAt to the past so GC picks it up
|
||||
await prisma.storageKey.updateMany({
|
||||
data: { createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000) },
|
||||
})
|
||||
|
||||
// Trigger GC job
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgeUnreferencedStorageKeys()
|
||||
|
||||
// Verify everything is wiped from DB
|
||||
expect(await prisma.project.findUnique({ where: { id: project.id } })).toBeNull()
|
||||
expect(await prisma.asset.findUnique({ where: { id: folderA.id } })).toBeNull()
|
||||
|
||||
@@ -39,7 +39,7 @@ describe('SkillService', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.zip',
|
||||
key: 'test.zip',
|
||||
storageKey: { create: { key: 'test.zip' } },
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
},
|
||||
|
||||
@@ -25,9 +25,11 @@ export class SkillService {
|
||||
} else if (req.assetId) {
|
||||
const asset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: req.assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (!asset || !asset.key) throw new Error('Asset not found or has no key')
|
||||
const { buffer } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', asset.key)
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) throw new Error('Asset not found or has no key')
|
||||
const { buffer } = await s3Service.getObject(process.env.S3_BUCKET || 'shumai', key)
|
||||
zipBuffer = buffer
|
||||
} else {
|
||||
throw new Error('Either assetId or githubUrl must be provided')
|
||||
@@ -68,7 +70,7 @@ export class SkillService {
|
||||
const skillAsset = await this.prismaClient.asset.create({
|
||||
data: {
|
||||
name: `${name}.zip`,
|
||||
key: assetKey,
|
||||
storageKey: { create: { key: assetKey } },
|
||||
type: 'file',
|
||||
mediaType: 'application/zip',
|
||||
sizeByte: repackedZip.length,
|
||||
|
||||
@@ -101,10 +101,15 @@ describe('UploadService', () => {
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
key: 'test-key',
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key' },
|
||||
create: { key: 'test-key' },
|
||||
},
|
||||
},
|
||||
mediaType: 'video/mp4',
|
||||
},
|
||||
})
|
||||
@@ -134,10 +139,15 @@ describe('UploadService', () => {
|
||||
data: {
|
||||
name: 'image.png',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
key: 'test-key',
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key' },
|
||||
create: { key: 'test-key' },
|
||||
},
|
||||
},
|
||||
mediaType: 'image/png',
|
||||
},
|
||||
})
|
||||
@@ -162,10 +172,15 @@ describe('UploadService', () => {
|
||||
data: {
|
||||
name: 'file.txt',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
key: 'test-key',
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key' },
|
||||
create: { key: 'test-key' },
|
||||
},
|
||||
},
|
||||
mediaType: 'text/plain',
|
||||
},
|
||||
})
|
||||
@@ -259,10 +274,15 @@ describe('UploadService', () => {
|
||||
data: {
|
||||
name: 'v2.txt',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId: stack.id,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: stack.id } },
|
||||
status: AssetStatus.uploading,
|
||||
key: 'v2-key',
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'v2-key' },
|
||||
create: { key: 'v2-key' },
|
||||
},
|
||||
},
|
||||
sizeByte: 2000,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -127,15 +127,15 @@ export class UploadService {
|
||||
data: {
|
||||
name: file.name,
|
||||
type: assetType,
|
||||
key,
|
||||
storageKey: key ? { create: { key } } : undefined,
|
||||
sortIndex: newSortIndex,
|
||||
mediaType: file.mediaType,
|
||||
status: AssetStatus.uploading,
|
||||
sizeByte: file.size,
|
||||
creatorId: userId,
|
||||
parentId,
|
||||
projectId,
|
||||
taskId,
|
||||
creator: userId ? { connect: { id: userId } } : undefined,
|
||||
parent: parentId ? { connect: { id: parentId } } : undefined,
|
||||
project: { connect: { id: projectId } },
|
||||
task: { connect: { id: taskId } },
|
||||
},
|
||||
})
|
||||
createdIds.push(newAsset.id)
|
||||
@@ -168,11 +168,15 @@ export class UploadService {
|
||||
): Promise<void> {
|
||||
const asset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: req.fileId },
|
||||
include: { project: { include: { team: true } } },
|
||||
include: {
|
||||
storageKey: true,
|
||||
project: { include: { team: true } },
|
||||
},
|
||||
})
|
||||
if (!asset) throw new Error('Asset not found')
|
||||
if (!asset.project) throw new Error('Project not found for asset')
|
||||
if (!asset.key) throw new Error('Asset has no key')
|
||||
const key = asset.storageKey?.key
|
||||
if (!key) throw new Error('Asset has no key')
|
||||
|
||||
if (req.errorMessage) {
|
||||
await this.prismaClient.asset.delete({ where: { id: asset.id } })
|
||||
@@ -192,7 +196,7 @@ export class UploadService {
|
||||
return
|
||||
}
|
||||
|
||||
const size = await s3Service.getObjectSize(process.env.S3_BUCKET || 'shumai', asset.key)
|
||||
const size = await s3Service.getObjectSize(process.env.S3_BUCKET || 'shumai', key)
|
||||
|
||||
const project = asset.project
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ describe('Transcode Activities', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'v.mp4',
|
||||
key: 'v.mp4',
|
||||
storageKey: { create: { key: 'v.mp4' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
},
|
||||
|
||||
@@ -255,10 +255,14 @@ export interface UpdateAssetMediaActivityParams {
|
||||
|
||||
export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityParams) {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const asset = await prisma.asset.findUnique({ where: { id: params.assetId } })
|
||||
if (!asset || !asset.key) throw new Error('Asset not found')
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: params.assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) throw new Error('Asset not found or has no key')
|
||||
|
||||
const infoKey = path.join(path.dirname(asset.key), 'info.json')
|
||||
const infoKey = path.join(path.dirname(key), 'info.json')
|
||||
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
|
||||
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ describe('Transcode Workflow', () => {
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
key: 'video.mp4',
|
||||
storageKey: { key: 'video.mp4' },
|
||||
mediaType: 'video/mp4',
|
||||
})
|
||||
|
||||
|
||||
@@ -38,14 +38,15 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
|
||||
// 2. Get Asset
|
||||
const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId)
|
||||
if (!asset || !asset.key) throw new Error('Asset not found')
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) throw new Error('Asset not found or has no key')
|
||||
|
||||
// 3. Get Worker-Specific Queue for Transcode
|
||||
workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity)
|
||||
|
||||
// 4. Download Media to Tmp
|
||||
const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
})
|
||||
const { filePath } = download
|
||||
tmpDir = download.tmpDir
|
||||
@@ -59,7 +60,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
})
|
||||
|
||||
mediaInfo.original = {
|
||||
key: asset.key,
|
||||
key,
|
||||
downloadUrl: '',
|
||||
filesizeInBytes: 0,
|
||||
codec: '',
|
||||
@@ -89,7 +90,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
|
||||
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
videoSpec,
|
||||
duration: metadata.duration,
|
||||
@@ -125,7 +126,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
if (imageSpec.width > metadata.originalWidth) continue
|
||||
|
||||
const imageTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
imageSpec,
|
||||
})
|
||||
@@ -143,7 +144,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
format: 'webp',
|
||||
}
|
||||
const imageTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
imageSpec,
|
||||
})
|
||||
@@ -164,7 +165,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
// 9. System: Thumbnail
|
||||
if (spec.thumbnail) {
|
||||
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
imageSpec: { width: 480, height: 0, quality: 80, format: 'webp' },
|
||||
})
|
||||
@@ -173,8 +174,8 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
|
||||
// 10. System: Sprite/Poster (Video only usually)
|
||||
if (spec.sprite || spec.poster) {
|
||||
const lastSlashIndex = asset.key.lastIndexOf('/')
|
||||
const assetDir = lastSlashIndex === -1 ? '' : asset.key.substring(0, lastSlashIndex)
|
||||
const lastSlashIndex = key.lastIndexOf('/')
|
||||
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
|
||||
|
||||
const spriteSpec: PrismaJson.SpriteInfo = {
|
||||
key: assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp',
|
||||
@@ -187,7 +188,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
|
||||
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
spriteSpec,
|
||||
posterSpec,
|
||||
|
||||
@@ -25,10 +25,10 @@ describe('AI Activities', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.png',
|
||||
key: 'test.png',
|
||||
storageKey: { create: { key: 'test.png' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
projectId: project.id,
|
||||
project: { connect: { id: project.id } },
|
||||
},
|
||||
})
|
||||
|
||||
@@ -54,7 +54,7 @@ describe('AI Activities', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.png',
|
||||
key: 'test.png',
|
||||
storageKey: { create: { key: 'test.png' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
},
|
||||
|
||||
@@ -11,6 +11,7 @@ export async function getAssetActivity(assetId: string) {
|
||||
return prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: {
|
||||
storageKey: true,
|
||||
project: {
|
||||
include: {
|
||||
team: true,
|
||||
@@ -46,7 +47,7 @@ export async function updateAssetMetadataActivity(params: UpdateAssetMetadataPar
|
||||
export async function getCommentActivity(commentId: string) {
|
||||
return prisma.assetComment.findUnique({
|
||||
where: { id: commentId },
|
||||
include: { attachments: { include: { asset: true } } },
|
||||
include: { attachments: { include: { asset: { include: { storageKey: true } } } } },
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { prisma } from '@/db'
|
||||
import { s3Service } from '@/services/s3/s3'
|
||||
import { logger } from '@/logger'
|
||||
|
||||
export async function purgeUnreferencedStorageKeysActivity(): Promise<{
|
||||
purgedCount: number
|
||||
physicalFilesDeleted: number
|
||||
}> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
let purgedCount = 0
|
||||
let physicalFilesDeleted = 0
|
||||
|
||||
// 1. Find storage keys with no associated assets and older than 24 hours
|
||||
// We use 24 hours to avoid race conditions with ongoing uploads or copies.
|
||||
const twentyFourHoursAgo = new Date()
|
||||
twentyFourHoursAgo.setDate(twentyFourHoursAgo.getDate() - 1)
|
||||
|
||||
const unreferencedKeys = await prisma.storageKey.findMany({
|
||||
where: {
|
||||
assets: { none: {} },
|
||||
createdAt: { lt: twentyFourHoursAgo },
|
||||
},
|
||||
take: 100,
|
||||
})
|
||||
|
||||
for (const sk of unreferencedKeys) {
|
||||
try {
|
||||
// Physically delete from S3
|
||||
// Handle complex keys (directories) same as before
|
||||
const parts = sk.key.split('/')
|
||||
if (parts.length > 2) {
|
||||
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
|
||||
const count = await s3Service.deletePrefix(bucket, prefix)
|
||||
physicalFilesDeleted += count
|
||||
} else {
|
||||
const count = await s3Service.deleteObject(bucket, sk.key)
|
||||
physicalFilesDeleted += count
|
||||
}
|
||||
|
||||
// Delete from database
|
||||
await prisma.storageKey.delete({
|
||||
where: { id: sk.id },
|
||||
})
|
||||
purgedCount++
|
||||
} catch (e: unknown) {
|
||||
logger.error(
|
||||
{ key: sk.key, error: e instanceof Error ? e.message : String(e) },
|
||||
'Failed to purge unreferenced storage key',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (purgedCount > 0) {
|
||||
logger.info(
|
||||
{ purgedCount, physicalFilesDeleted },
|
||||
`Garbage collection: purged ${purgedCount} storage keys and ${physicalFilesDeleted} physical files`,
|
||||
)
|
||||
}
|
||||
|
||||
return { purgedCount, physicalFilesDeleted }
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import * as aiActivities from './ai'
|
||||
import * as transcodeActivities from '@/transcode/activities/transcode'
|
||||
import * as agentActivities from './agent'
|
||||
import * as dbActivities from './db'
|
||||
import * as gcActivities from './gc'
|
||||
|
||||
export const activities = {
|
||||
...taskActivities,
|
||||
@@ -10,4 +11,5 @@ export const activities = {
|
||||
...transcodeActivities,
|
||||
...agentActivities,
|
||||
...dbActivities,
|
||||
...gcActivities,
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ describe('Task Activities', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.mp4',
|
||||
key: 'test.mp4',
|
||||
storageKey: { create: { key: 'test.mp4' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
},
|
||||
|
||||
@@ -56,10 +56,10 @@ describe('WorkflowService', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.mp3',
|
||||
key: 'test/test.mp3',
|
||||
storageKey: { create: { key: 'test/test.mp3' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
projectId: project.id,
|
||||
project: { connect: { id: project.id } },
|
||||
},
|
||||
})
|
||||
|
||||
@@ -100,11 +100,11 @@ describe('WorkflowService', () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.png',
|
||||
key: 'test/test.png',
|
||||
storageKey: { create: { key: 'test/test.png' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
project: { connect: { id: project.id } },
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ describe('AIAutofill Workflow', () => {
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
key: 'test.mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
mediaType: 'video/mp4',
|
||||
project: { id: 'proj-1', teamId: 'team-1' },
|
||||
})
|
||||
@@ -140,7 +140,7 @@ describe('AIAutofill Workflow', () => {
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
key: 'test.mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
mediaType: 'video/mp4',
|
||||
project: { id: 'proj-1', teamId: 'team-1' },
|
||||
})
|
||||
|
||||
@@ -47,7 +47,8 @@ export async function aiAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
|
||||
// 1. Get Asset
|
||||
const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId)
|
||||
if (!asset || !asset.project) {
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !asset.project || !key) {
|
||||
throw new Error('Asset or project not found')
|
||||
}
|
||||
const teamId = asset.project.teamId
|
||||
@@ -63,13 +64,13 @@ export async function aiAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
)
|
||||
|
||||
const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, {
|
||||
assetKey: asset.key || '',
|
||||
assetKey: key,
|
||||
})
|
||||
const { filePath } = download
|
||||
tmpDir = download.tmpDir
|
||||
|
||||
const generatedFiles = await executeActivity(transcodeWorkerQueue, extractAiMetadataActivity, {
|
||||
assetKey: asset.key || '',
|
||||
assetKey: key,
|
||||
filePath,
|
||||
type: 'autofill',
|
||||
isImage,
|
||||
|
||||
@@ -42,7 +42,8 @@ export async function aiTranscriptionMedia(task: WorkflowTask): Promise<void> {
|
||||
|
||||
// 1. Get Asset
|
||||
const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId)
|
||||
if (!asset || !asset.key) {
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) {
|
||||
throw new Error('Asset not found')
|
||||
}
|
||||
|
||||
@@ -53,13 +54,13 @@ export async function aiTranscriptionMedia(task: WorkflowTask): Promise<void> {
|
||||
)
|
||||
|
||||
const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
})
|
||||
const { filePath } = download
|
||||
tmpDir = download.tmpDir
|
||||
|
||||
const generatedFiles = await executeActivity(transcodeWorkerQueue, extractAiMetadataActivity, {
|
||||
assetKey: asset.key,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
type: 'transcription',
|
||||
isImage: false,
|
||||
|
||||
Reference in New Issue
Block a user