refactor(asset): optimize cleanup job and solidify GC with locking

This commit is contained in:
yiling
2026-05-25 09:57:46 +08:00
parent 5dc3bdb38e
commit 2e09ce6ab5
9 changed files with 147 additions and 85 deletions
+9 -4
View File
@@ -42,6 +42,10 @@ enum AssetStatus {
processed
trashed
pending_purge
}
enum StorageKeyStatus {
active
purging
}
@@ -406,10 +410,11 @@ model Asset {
}
model StorageKey {
id String @id @default(ulid())
key String @unique
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
id String @id @default(ulid())
key String @unique
status StorageKeyStatus @default(active)
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
assets Asset[]
@@map("storage_keys")
+34
View File
@@ -328,6 +328,23 @@ export type EnumAssetStatusWithAggregatesFilter<$PrismaModel = never> = {
_max?: Prisma.NestedEnumAssetStatusFilter<$PrismaModel>
}
export type EnumStorageKeyStatusFilter<$PrismaModel = never> = {
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
not?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel> | $Enums.StorageKeyStatus
}
export type EnumStorageKeyStatusWithAggregatesFilter<$PrismaModel = never> = {
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
not?: Prisma.NestedEnumStorageKeyStatusWithAggregatesFilter<$PrismaModel> | $Enums.StorageKeyStatus
_count?: Prisma.NestedIntFilter<$PrismaModel>
_min?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
_max?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
}
export type FloatNullableFilter<$PrismaModel = never> = {
equals?: number | Prisma.FloatFieldRefInput<$PrismaModel> | null
in?: number[] | Prisma.ListFloatFieldRefInput<$PrismaModel> | null
@@ -909,6 +926,23 @@ export type NestedEnumAssetStatusWithAggregatesFilter<$PrismaModel = never> = {
_max?: Prisma.NestedEnumAssetStatusFilter<$PrismaModel>
}
export type NestedEnumStorageKeyStatusFilter<$PrismaModel = never> = {
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
not?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel> | $Enums.StorageKeyStatus
}
export type NestedEnumStorageKeyStatusWithAggregatesFilter<$PrismaModel = never> = {
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
not?: Prisma.NestedEnumStorageKeyStatusWithAggregatesFilter<$PrismaModel> | $Enums.StorageKeyStatus
_count?: Prisma.NestedIntFilter<$PrismaModel>
_min?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
_max?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
}
export type NestedFloatNullableFilter<$PrismaModel = never> = {
equals?: number | Prisma.FloatFieldRefInput<$PrismaModel> | null
in?: number[] | Prisma.ListFloatFieldRefInput<$PrismaModel> | null
+9 -2
View File
@@ -37,13 +37,20 @@ export const AssetStatus = {
processing: 'processing',
processed: 'processed',
trashed: 'trashed',
pending_purge: 'pending_purge',
purging: 'purging'
pending_purge: 'pending_purge'
} as const
export type AssetStatus = (typeof AssetStatus)[keyof typeof AssetStatus]
export const StorageKeyStatus = {
active: 'active',
purging: 'purging'
} as const
export type StorageKeyStatus = (typeof StorageKeyStatus)[keyof typeof StorageKeyStatus]
export const InviteRole = {
editor: 'editor',
reviewer: 'reviewer'
File diff suppressed because one or more lines are too long
+15
View File
@@ -2964,6 +2964,7 @@ export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof As
export const StorageKeyScalarFieldEnum = {
id: 'id',
key: 'key',
status: 'status',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
@@ -3406,6 +3407,20 @@ export type ListEnumAssetStatusFieldRefInput<$PrismaModel> = FieldRefInputType<$
/**
* Reference to a field of type 'StorageKeyStatus'
*/
export type EnumStorageKeyStatusFieldRefInput<$PrismaModel> = FieldRefInputType<$PrismaModel, 'StorageKeyStatus'>
/**
* Reference to a field of type 'StorageKeyStatus[]'
*/
export type ListEnumStorageKeyStatusFieldRefInput<$PrismaModel> = FieldRefInputType<$PrismaModel, 'StorageKeyStatus[]'>
/**
* Reference to a field of type 'Float'
*/
+1
View File
@@ -297,6 +297,7 @@ export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof As
export const StorageKeyScalarFieldEnum = {
id: 'id',
key: 'key',
status: 'status',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
+37 -1
View File
@@ -29,6 +29,7 @@ export type AggregateStorageKey = {
export type StorageKeyMinAggregateOutputType = {
id: string | null
key: string | null
status: $Enums.StorageKeyStatus | null
createdAt: Date | null
updatedAt: Date | null
}
@@ -36,6 +37,7 @@ export type StorageKeyMinAggregateOutputType = {
export type StorageKeyMaxAggregateOutputType = {
id: string | null
key: string | null
status: $Enums.StorageKeyStatus | null
createdAt: Date | null
updatedAt: Date | null
}
@@ -43,6 +45,7 @@ export type StorageKeyMaxAggregateOutputType = {
export type StorageKeyCountAggregateOutputType = {
id: number
key: number
status: number
createdAt: number
updatedAt: number
_all: number
@@ -52,6 +55,7 @@ export type StorageKeyCountAggregateOutputType = {
export type StorageKeyMinAggregateInputType = {
id?: true
key?: true
status?: true
createdAt?: true
updatedAt?: true
}
@@ -59,6 +63,7 @@ export type StorageKeyMinAggregateInputType = {
export type StorageKeyMaxAggregateInputType = {
id?: true
key?: true
status?: true
createdAt?: true
updatedAt?: true
}
@@ -66,6 +71,7 @@ export type StorageKeyMaxAggregateInputType = {
export type StorageKeyCountAggregateInputType = {
id?: true
key?: true
status?: true
createdAt?: true
updatedAt?: true
_all?: true
@@ -146,6 +152,7 @@ export type StorageKeyGroupByArgs<ExtArgs extends runtime.Types.Extensions.Inter
export type StorageKeyGroupByOutputType = {
id: string
key: string
status: $Enums.StorageKeyStatus
createdAt: Date
updatedAt: Date
_count: StorageKeyCountAggregateOutputType | null
@@ -174,6 +181,7 @@ export type StorageKeyWhereInput = {
NOT?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
id?: Prisma.StringFilter<"StorageKey"> | string
key?: Prisma.StringFilter<"StorageKey"> | string
status?: Prisma.EnumStorageKeyStatusFilter<"StorageKey"> | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
updatedAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
assets?: Prisma.AssetListRelationFilter
@@ -182,6 +190,7 @@ export type StorageKeyWhereInput = {
export type StorageKeyOrderByWithRelationInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
assets?: Prisma.AssetOrderByRelationAggregateInput
@@ -193,6 +202,7 @@ export type StorageKeyWhereUniqueInput = Prisma.AtLeast<{
AND?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
OR?: Prisma.StorageKeyWhereInput[]
NOT?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
status?: Prisma.EnumStorageKeyStatusFilter<"StorageKey"> | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
updatedAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
assets?: Prisma.AssetListRelationFilter
@@ -201,6 +211,7 @@ export type StorageKeyWhereUniqueInput = Prisma.AtLeast<{
export type StorageKeyOrderByWithAggregationInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
_count?: Prisma.StorageKeyCountOrderByAggregateInput
@@ -214,6 +225,7 @@ export type StorageKeyScalarWhereWithAggregatesInput = {
NOT?: Prisma.StorageKeyScalarWhereWithAggregatesInput | Prisma.StorageKeyScalarWhereWithAggregatesInput[]
id?: Prisma.StringWithAggregatesFilter<"StorageKey"> | string
key?: Prisma.StringWithAggregatesFilter<"StorageKey"> | string
status?: Prisma.EnumStorageKeyStatusWithAggregatesFilter<"StorageKey"> | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeWithAggregatesFilter<"StorageKey"> | Date | string
updatedAt?: Prisma.DateTimeWithAggregatesFilter<"StorageKey"> | Date | string
}
@@ -221,6 +233,7 @@ export type StorageKeyScalarWhereWithAggregatesInput = {
export type StorageKeyCreateInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
createdAt?: Date | string
updatedAt?: Date | string
assets?: Prisma.AssetCreateNestedManyWithoutStorageKeyInput
@@ -229,6 +242,7 @@ export type StorageKeyCreateInput = {
export type StorageKeyUncheckedCreateInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
createdAt?: Date | string
updatedAt?: Date | string
assets?: Prisma.AssetUncheckedCreateNestedManyWithoutStorageKeyInput
@@ -237,6 +251,7 @@ export type StorageKeyUncheckedCreateInput = {
export type StorageKeyUpdateInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
assets?: Prisma.AssetUpdateManyWithoutStorageKeyNestedInput
@@ -245,6 +260,7 @@ export type StorageKeyUpdateInput = {
export type StorageKeyUncheckedUpdateInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
assets?: Prisma.AssetUncheckedUpdateManyWithoutStorageKeyNestedInput
@@ -253,6 +269,7 @@ export type StorageKeyUncheckedUpdateInput = {
export type StorageKeyCreateManyInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
createdAt?: Date | string
updatedAt?: Date | string
}
@@ -260,6 +277,7 @@ export type StorageKeyCreateManyInput = {
export type StorageKeyUpdateManyMutationInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -267,6 +285,7 @@ export type StorageKeyUpdateManyMutationInput = {
export type StorageKeyUncheckedUpdateManyInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -279,6 +298,7 @@ export type StorageKeyNullableScalarRelationFilter = {
export type StorageKeyCountOrderByAggregateInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
}
@@ -286,6 +306,7 @@ export type StorageKeyCountOrderByAggregateInput = {
export type StorageKeyMaxOrderByAggregateInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
}
@@ -293,6 +314,7 @@ export type StorageKeyMaxOrderByAggregateInput = {
export type StorageKeyMinOrderByAggregateInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
}
@@ -313,9 +335,14 @@ export type StorageKeyUpdateOneWithoutAssetsNestedInput = {
update?: Prisma.XOR<Prisma.XOR<Prisma.StorageKeyUpdateToOneWithWhereWithoutAssetsInput, Prisma.StorageKeyUpdateWithoutAssetsInput>, Prisma.StorageKeyUncheckedUpdateWithoutAssetsInput>
}
export type EnumStorageKeyStatusFieldUpdateOperationsInput = {
set?: $Enums.StorageKeyStatus
}
export type StorageKeyCreateWithoutAssetsInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
createdAt?: Date | string
updatedAt?: Date | string
}
@@ -323,6 +350,7 @@ export type StorageKeyCreateWithoutAssetsInput = {
export type StorageKeyUncheckedCreateWithoutAssetsInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
createdAt?: Date | string
updatedAt?: Date | string
}
@@ -346,6 +374,7 @@ export type StorageKeyUpdateToOneWithWhereWithoutAssetsInput = {
export type StorageKeyUpdateWithoutAssetsInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -353,6 +382,7 @@ export type StorageKeyUpdateWithoutAssetsInput = {
export type StorageKeyUncheckedUpdateWithoutAssetsInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -391,6 +421,7 @@ export type StorageKeyCountOutputTypeCountAssetsArgs<ExtArgs extends runtime.Typ
export type StorageKeySelect<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetSelect<{
id?: boolean
key?: boolean
status?: boolean
createdAt?: boolean
updatedAt?: boolean
assets?: boolean | Prisma.StorageKey$assetsArgs<ExtArgs>
@@ -400,6 +431,7 @@ export type StorageKeySelect<ExtArgs extends runtime.Types.Extensions.InternalAr
export type StorageKeySelectCreateManyAndReturn<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetSelect<{
id?: boolean
key?: boolean
status?: boolean
createdAt?: boolean
updatedAt?: boolean
}, ExtArgs["result"]["storageKey"]>
@@ -407,6 +439,7 @@ export type StorageKeySelectCreateManyAndReturn<ExtArgs extends runtime.Types.Ex
export type StorageKeySelectUpdateManyAndReturn<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetSelect<{
id?: boolean
key?: boolean
status?: boolean
createdAt?: boolean
updatedAt?: boolean
}, ExtArgs["result"]["storageKey"]>
@@ -414,11 +447,12 @@ export type StorageKeySelectUpdateManyAndReturn<ExtArgs extends runtime.Types.Ex
export type StorageKeySelectScalar = {
id?: boolean
key?: boolean
status?: boolean
createdAt?: boolean
updatedAt?: boolean
}
export type StorageKeyOmit<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetOmit<"id" | "key" | "createdAt" | "updatedAt", ExtArgs["result"]["storageKey"]>
export type StorageKeyOmit<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetOmit<"id" | "key" | "status" | "createdAt" | "updatedAt", ExtArgs["result"]["storageKey"]>
export type StorageKeyInclude<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = {
assets?: boolean | Prisma.StorageKey$assetsArgs<ExtArgs>
_count?: boolean | Prisma.StorageKeyCountOutputTypeDefaultArgs<ExtArgs>
@@ -434,6 +468,7 @@ export type $StorageKeyPayload<ExtArgs extends runtime.Types.Extensions.Internal
scalars: runtime.Types.Extensions.GetPayloadResult<{
id: string
key: string
status: $Enums.StorageKeyStatus
createdAt: Date
updatedAt: Date
}, ExtArgs["result"]["storageKey"]>
@@ -862,6 +897,7 @@ export interface Prisma__StorageKeyClient<T, Null = never, ExtArgs extends runti
export interface StorageKeyFieldRefs {
readonly id: Prisma.FieldRef<"StorageKey", 'String'>
readonly key: Prisma.FieldRef<"StorageKey", 'String'>
readonly status: Prisma.FieldRef<"StorageKey", 'StorageKeyStatus'>
readonly createdAt: Prisma.FieldRef<"StorageKey", 'DateTime'>
readonly updatedAt: Prisma.FieldRef<"StorageKey", 'DateTime'>
}
+1 -5
View File
@@ -1311,11 +1311,7 @@ describe('AssetService', () => {
})
}
// 3. Trigger the purge job for the FIRST batch (100 items)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgePendingAssets()
// ...
// 3. Trigger the purge job
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).purgePendingAssets()
+37 -69
View File
@@ -18,7 +18,7 @@ import {
UpdateAssetOrderRequest,
UserInfo,
} from '@/dtos/asset'
import { Asset, AssetType, Prisma, StorageKey } from '@/generated/prisma/client.ts'
import { Asset, AssetStatus, AssetType, Prisma, StorageKey, StorageKeyStatus } from '@/generated/prisma/client.ts'
import { PaginatedData, paginateQuery, PaginationParams } from '@/services/pagination'
import { s3Service } from '@/services/s3/s3'
import { generateKeyBetween } from 'jittered-fractional-indexing'
@@ -980,45 +980,26 @@ export class AssetService {
private cleanupJobsRunning = false
startCleanupJob() {
async startCleanupJob() {
this.cleanupJobsRunning = true
// Stage 1: Expiration Job - Move expired trashed roots to pending_purge
const runExpiration = async () => {
const runCleanup = async () => {
if (!this.cleanupJobsRunning) return
try {
await this.expireTrashedAssets()
} catch (e: unknown) {
console.error('Error in asset expiration job:', e)
}
setTimeout(runExpiration, 5000) // Run every 5 seconds, less frequent
}
// Stage 2: Physical Purge Job - Delete files from S3 and wipe from DB
const runPurge = async () => {
if (!this.cleanupJobsRunning) return
try {
await this.purgePendingAssets()
} catch (e: unknown) {
console.error('Error in asset purge job:', e)
}
setTimeout(runPurge, 1000) // Run every 1 second
}
// Stage 3: Garbage Collection Job - Purge unreferenced StorageKeys and physical files
const runGc = async () => {
if (!this.cleanupJobsRunning) return
try {
await this.purgeUnreferencedStorageKeys()
} catch (e: unknown) {
console.error('Error in asset GC job:', e)
console.error('Error in asset cleanup job:', e)
}
if (this.cleanupJobsRunning) {
setTimeout(runCleanup, 5000) // Run every 5 seconds after previous run completion
}
setTimeout(runGc, 10000) // Run every 10 seconds
}
runExpiration()
runPurge()
runGc()
runCleanup()
}
stopCleanupJob() {
@@ -1062,39 +1043,19 @@ export class AssetService {
}
/**
* Stage 2: Physically delete files and DB records for assets marked as 'pending_purge'.
* Stage 2: Delete DB records for assets marked as 'pending_purge'.
*/
private async purgePendingAssets() {
// 1. Lock 100 assets using raw SQL FOR UPDATE SKIP LOCKED and mark them as 'purging'
// We include a timeout check (1 hour) to allow picking up stalled tasks.
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string }[]>`
WITH to_purge AS (
SELECT id FROM assets
WHERE status = 'pending_purge'
OR (status = 'purging' AND updated_at < NOW() - INTERVAL '1 hour')
LIMIT 100
FOR UPDATE SKIP LOCKED
)
UPDATE assets SET status = 'purging', updated_at = NOW()
WHERE id IN (SELECT id FROM to_purge)
RETURNING id;
`
if (lockedAssets.length === 0) return
const successfulIds = lockedAssets.map((a) => a.id)
// 3. Hard delete from database
if (successfulIds.length > 0) {
await this.prismaClient.asset.deleteMany({
where: { id: { in: successfulIds } },
})
const { count } = await this.prismaClient.asset.deleteMany({
where: { status: AssetStatus.pending_purge },
})
if (count > 0) {
logger.info(
{
purgedCount: successfulIds.length,
purgedCount: count,
},
`${successfulIds.length} assets purged from database`,
`${count} assets purged from database`,
)
}
}
@@ -1105,23 +1066,30 @@ export class AssetService {
*/
private async purgeUnreferencedStorageKeys() {
const bucket = process.env.S3_BUCKET || 'shumai'
let purgedCount = 0
let physicalFilesDeleted = 0
// 1. Find storage keys with no associated assets and older than 24 hours
// We use 24 hours to avoid race conditions with ongoing uploads or copies.
const twentyFourHoursAgo = new Date()
twentyFourHoursAgo.setDate(twentyFourHoursAgo.getDate() - 1)
// using FOR UPDATE SKIP LOCKED to ensure multiple servers don't pick up the same keys.
const lockedKeys = await this.prismaClient.$queryRaw<{ id: string; key: string }[]>`
WITH to_purge AS (
SELECT sk.id FROM storage_keys sk
LEFT JOIN assets a ON a.storage_key_id = sk.id
WHERE a.id IS NULL
AND sk.created_at < NOW() - INTERVAL '24 hours'
AND (sk.status = 'active' OR (sk.status = 'purging' AND sk.updated_at < NOW() - INTERVAL '1 hour'))
LIMIT 100
FOR UPDATE SKIP LOCKED
)
UPDATE storage_keys SET status = 'purging', updated_at = NOW()
WHERE id IN (SELECT id FROM to_purge)
RETURNING id, key;
`
const unreferencedKeys = await this.prismaClient.storageKey.findMany({
where: {
assets: { none: {} },
createdAt: { lt: twentyFourHoursAgo },
},
take: 100,
})
if (lockedKeys.length === 0) return
for (const sk of unreferencedKeys) {
let physicalFilesDeleted = 0
let purgedCount = 0
for (const sk of lockedKeys) {
try {
// Physically delete from S3
const parts = sk.key.split('/')