mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 12:17:17 +00:00
refactor(asset): optimize cleanup job and solidify GC with locking
This commit is contained in:
@@ -42,6 +42,10 @@ enum AssetStatus {
|
||||
processed
|
||||
trashed
|
||||
pending_purge
|
||||
}
|
||||
|
||||
enum StorageKeyStatus {
|
||||
active
|
||||
purging
|
||||
}
|
||||
|
||||
@@ -406,10 +410,11 @@ model Asset {
|
||||
}
|
||||
|
||||
model StorageKey {
|
||||
id String @id @default(ulid())
|
||||
key String @unique
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
id String @id @default(ulid())
|
||||
key String @unique
|
||||
status StorageKeyStatus @default(active)
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
assets Asset[]
|
||||
|
||||
@@map("storage_keys")
|
||||
|
||||
Generated
+34
@@ -328,6 +328,23 @@ export type EnumAssetStatusWithAggregatesFilter<$PrismaModel = never> = {
|
||||
_max?: Prisma.NestedEnumAssetStatusFilter<$PrismaModel>
|
||||
}
|
||||
|
||||
export type EnumStorageKeyStatusFilter<$PrismaModel = never> = {
|
||||
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
not?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel> | $Enums.StorageKeyStatus
|
||||
}
|
||||
|
||||
export type EnumStorageKeyStatusWithAggregatesFilter<$PrismaModel = never> = {
|
||||
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
not?: Prisma.NestedEnumStorageKeyStatusWithAggregatesFilter<$PrismaModel> | $Enums.StorageKeyStatus
|
||||
_count?: Prisma.NestedIntFilter<$PrismaModel>
|
||||
_min?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
|
||||
_max?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
|
||||
}
|
||||
|
||||
export type FloatNullableFilter<$PrismaModel = never> = {
|
||||
equals?: number | Prisma.FloatFieldRefInput<$PrismaModel> | null
|
||||
in?: number[] | Prisma.ListFloatFieldRefInput<$PrismaModel> | null
|
||||
@@ -909,6 +926,23 @@ export type NestedEnumAssetStatusWithAggregatesFilter<$PrismaModel = never> = {
|
||||
_max?: Prisma.NestedEnumAssetStatusFilter<$PrismaModel>
|
||||
}
|
||||
|
||||
export type NestedEnumStorageKeyStatusFilter<$PrismaModel = never> = {
|
||||
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
not?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel> | $Enums.StorageKeyStatus
|
||||
}
|
||||
|
||||
export type NestedEnumStorageKeyStatusWithAggregatesFilter<$PrismaModel = never> = {
|
||||
equals?: $Enums.StorageKeyStatus | Prisma.EnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
in?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
notIn?: $Enums.StorageKeyStatus[] | Prisma.ListEnumStorageKeyStatusFieldRefInput<$PrismaModel>
|
||||
not?: Prisma.NestedEnumStorageKeyStatusWithAggregatesFilter<$PrismaModel> | $Enums.StorageKeyStatus
|
||||
_count?: Prisma.NestedIntFilter<$PrismaModel>
|
||||
_min?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
|
||||
_max?: Prisma.NestedEnumStorageKeyStatusFilter<$PrismaModel>
|
||||
}
|
||||
|
||||
export type NestedFloatNullableFilter<$PrismaModel = never> = {
|
||||
equals?: number | Prisma.FloatFieldRefInput<$PrismaModel> | null
|
||||
in?: number[] | Prisma.ListFloatFieldRefInput<$PrismaModel> | null
|
||||
|
||||
Generated
+9
-2
@@ -37,13 +37,20 @@ export const AssetStatus = {
|
||||
processing: 'processing',
|
||||
processed: 'processed',
|
||||
trashed: 'trashed',
|
||||
pending_purge: 'pending_purge',
|
||||
purging: 'purging'
|
||||
pending_purge: 'pending_purge'
|
||||
} as const
|
||||
|
||||
export type AssetStatus = (typeof AssetStatus)[keyof typeof AssetStatus]
|
||||
|
||||
|
||||
export const StorageKeyStatus = {
|
||||
active: 'active',
|
||||
purging: 'purging'
|
||||
} as const
|
||||
|
||||
export type StorageKeyStatus = (typeof StorageKeyStatus)[keyof typeof StorageKeyStatus]
|
||||
|
||||
|
||||
export const InviteRole = {
|
||||
editor: 'editor',
|
||||
reviewer: 'reviewer'
|
||||
|
||||
Generated
+4
-4
File diff suppressed because one or more lines are too long
+15
@@ -2964,6 +2964,7 @@ export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof As
|
||||
export const StorageKeyScalarFieldEnum = {
|
||||
id: 'id',
|
||||
key: 'key',
|
||||
status: 'status',
|
||||
createdAt: 'createdAt',
|
||||
updatedAt: 'updatedAt'
|
||||
} as const
|
||||
@@ -3406,6 +3407,20 @@ export type ListEnumAssetStatusFieldRefInput<$PrismaModel> = FieldRefInputType<$
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Reference to a field of type 'StorageKeyStatus'
|
||||
*/
|
||||
export type EnumStorageKeyStatusFieldRefInput<$PrismaModel> = FieldRefInputType<$PrismaModel, 'StorageKeyStatus'>
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Reference to a field of type 'StorageKeyStatus[]'
|
||||
*/
|
||||
export type ListEnumStorageKeyStatusFieldRefInput<$PrismaModel> = FieldRefInputType<$PrismaModel, 'StorageKeyStatus[]'>
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Reference to a field of type 'Float'
|
||||
*/
|
||||
|
||||
@@ -297,6 +297,7 @@ export type AssetScalarFieldEnum = (typeof AssetScalarFieldEnum)[keyof typeof As
|
||||
export const StorageKeyScalarFieldEnum = {
|
||||
id: 'id',
|
||||
key: 'key',
|
||||
status: 'status',
|
||||
createdAt: 'createdAt',
|
||||
updatedAt: 'updatedAt'
|
||||
} as const
|
||||
|
||||
+37
-1
@@ -29,6 +29,7 @@ export type AggregateStorageKey = {
|
||||
export type StorageKeyMinAggregateOutputType = {
|
||||
id: string | null
|
||||
key: string | null
|
||||
status: $Enums.StorageKeyStatus | null
|
||||
createdAt: Date | null
|
||||
updatedAt: Date | null
|
||||
}
|
||||
@@ -36,6 +37,7 @@ export type StorageKeyMinAggregateOutputType = {
|
||||
export type StorageKeyMaxAggregateOutputType = {
|
||||
id: string | null
|
||||
key: string | null
|
||||
status: $Enums.StorageKeyStatus | null
|
||||
createdAt: Date | null
|
||||
updatedAt: Date | null
|
||||
}
|
||||
@@ -43,6 +45,7 @@ export type StorageKeyMaxAggregateOutputType = {
|
||||
export type StorageKeyCountAggregateOutputType = {
|
||||
id: number
|
||||
key: number
|
||||
status: number
|
||||
createdAt: number
|
||||
updatedAt: number
|
||||
_all: number
|
||||
@@ -52,6 +55,7 @@ export type StorageKeyCountAggregateOutputType = {
|
||||
export type StorageKeyMinAggregateInputType = {
|
||||
id?: true
|
||||
key?: true
|
||||
status?: true
|
||||
createdAt?: true
|
||||
updatedAt?: true
|
||||
}
|
||||
@@ -59,6 +63,7 @@ export type StorageKeyMinAggregateInputType = {
|
||||
export type StorageKeyMaxAggregateInputType = {
|
||||
id?: true
|
||||
key?: true
|
||||
status?: true
|
||||
createdAt?: true
|
||||
updatedAt?: true
|
||||
}
|
||||
@@ -66,6 +71,7 @@ export type StorageKeyMaxAggregateInputType = {
|
||||
export type StorageKeyCountAggregateInputType = {
|
||||
id?: true
|
||||
key?: true
|
||||
status?: true
|
||||
createdAt?: true
|
||||
updatedAt?: true
|
||||
_all?: true
|
||||
@@ -146,6 +152,7 @@ export type StorageKeyGroupByArgs<ExtArgs extends runtime.Types.Extensions.Inter
|
||||
export type StorageKeyGroupByOutputType = {
|
||||
id: string
|
||||
key: string
|
||||
status: $Enums.StorageKeyStatus
|
||||
createdAt: Date
|
||||
updatedAt: Date
|
||||
_count: StorageKeyCountAggregateOutputType | null
|
||||
@@ -174,6 +181,7 @@ export type StorageKeyWhereInput = {
|
||||
NOT?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
|
||||
id?: Prisma.StringFilter<"StorageKey"> | string
|
||||
key?: Prisma.StringFilter<"StorageKey"> | string
|
||||
status?: Prisma.EnumStorageKeyStatusFilter<"StorageKey"> | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
|
||||
updatedAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
|
||||
assets?: Prisma.AssetListRelationFilter
|
||||
@@ -182,6 +190,7 @@ export type StorageKeyWhereInput = {
|
||||
export type StorageKeyOrderByWithRelationInput = {
|
||||
id?: Prisma.SortOrder
|
||||
key?: Prisma.SortOrder
|
||||
status?: Prisma.SortOrder
|
||||
createdAt?: Prisma.SortOrder
|
||||
updatedAt?: Prisma.SortOrder
|
||||
assets?: Prisma.AssetOrderByRelationAggregateInput
|
||||
@@ -193,6 +202,7 @@ export type StorageKeyWhereUniqueInput = Prisma.AtLeast<{
|
||||
AND?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
|
||||
OR?: Prisma.StorageKeyWhereInput[]
|
||||
NOT?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
|
||||
status?: Prisma.EnumStorageKeyStatusFilter<"StorageKey"> | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
|
||||
updatedAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
|
||||
assets?: Prisma.AssetListRelationFilter
|
||||
@@ -201,6 +211,7 @@ export type StorageKeyWhereUniqueInput = Prisma.AtLeast<{
|
||||
export type StorageKeyOrderByWithAggregationInput = {
|
||||
id?: Prisma.SortOrder
|
||||
key?: Prisma.SortOrder
|
||||
status?: Prisma.SortOrder
|
||||
createdAt?: Prisma.SortOrder
|
||||
updatedAt?: Prisma.SortOrder
|
||||
_count?: Prisma.StorageKeyCountOrderByAggregateInput
|
||||
@@ -214,6 +225,7 @@ export type StorageKeyScalarWhereWithAggregatesInput = {
|
||||
NOT?: Prisma.StorageKeyScalarWhereWithAggregatesInput | Prisma.StorageKeyScalarWhereWithAggregatesInput[]
|
||||
id?: Prisma.StringWithAggregatesFilter<"StorageKey"> | string
|
||||
key?: Prisma.StringWithAggregatesFilter<"StorageKey"> | string
|
||||
status?: Prisma.EnumStorageKeyStatusWithAggregatesFilter<"StorageKey"> | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeWithAggregatesFilter<"StorageKey"> | Date | string
|
||||
updatedAt?: Prisma.DateTimeWithAggregatesFilter<"StorageKey"> | Date | string
|
||||
}
|
||||
@@ -221,6 +233,7 @@ export type StorageKeyScalarWhereWithAggregatesInput = {
|
||||
export type StorageKeyCreateInput = {
|
||||
id?: string
|
||||
key: string
|
||||
status?: $Enums.StorageKeyStatus
|
||||
createdAt?: Date | string
|
||||
updatedAt?: Date | string
|
||||
assets?: Prisma.AssetCreateNestedManyWithoutStorageKeyInput
|
||||
@@ -229,6 +242,7 @@ export type StorageKeyCreateInput = {
|
||||
export type StorageKeyUncheckedCreateInput = {
|
||||
id?: string
|
||||
key: string
|
||||
status?: $Enums.StorageKeyStatus
|
||||
createdAt?: Date | string
|
||||
updatedAt?: Date | string
|
||||
assets?: Prisma.AssetUncheckedCreateNestedManyWithoutStorageKeyInput
|
||||
@@ -237,6 +251,7 @@ export type StorageKeyUncheckedCreateInput = {
|
||||
export type StorageKeyUpdateInput = {
|
||||
id?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
key?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
assets?: Prisma.AssetUpdateManyWithoutStorageKeyNestedInput
|
||||
@@ -245,6 +260,7 @@ export type StorageKeyUpdateInput = {
|
||||
export type StorageKeyUncheckedUpdateInput = {
|
||||
id?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
key?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
assets?: Prisma.AssetUncheckedUpdateManyWithoutStorageKeyNestedInput
|
||||
@@ -253,6 +269,7 @@ export type StorageKeyUncheckedUpdateInput = {
|
||||
export type StorageKeyCreateManyInput = {
|
||||
id?: string
|
||||
key: string
|
||||
status?: $Enums.StorageKeyStatus
|
||||
createdAt?: Date | string
|
||||
updatedAt?: Date | string
|
||||
}
|
||||
@@ -260,6 +277,7 @@ export type StorageKeyCreateManyInput = {
|
||||
export type StorageKeyUpdateManyMutationInput = {
|
||||
id?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
key?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
}
|
||||
@@ -267,6 +285,7 @@ export type StorageKeyUpdateManyMutationInput = {
|
||||
export type StorageKeyUncheckedUpdateManyInput = {
|
||||
id?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
key?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
}
|
||||
@@ -279,6 +298,7 @@ export type StorageKeyNullableScalarRelationFilter = {
|
||||
export type StorageKeyCountOrderByAggregateInput = {
|
||||
id?: Prisma.SortOrder
|
||||
key?: Prisma.SortOrder
|
||||
status?: Prisma.SortOrder
|
||||
createdAt?: Prisma.SortOrder
|
||||
updatedAt?: Prisma.SortOrder
|
||||
}
|
||||
@@ -286,6 +306,7 @@ export type StorageKeyCountOrderByAggregateInput = {
|
||||
export type StorageKeyMaxOrderByAggregateInput = {
|
||||
id?: Prisma.SortOrder
|
||||
key?: Prisma.SortOrder
|
||||
status?: Prisma.SortOrder
|
||||
createdAt?: Prisma.SortOrder
|
||||
updatedAt?: Prisma.SortOrder
|
||||
}
|
||||
@@ -293,6 +314,7 @@ export type StorageKeyMaxOrderByAggregateInput = {
|
||||
export type StorageKeyMinOrderByAggregateInput = {
|
||||
id?: Prisma.SortOrder
|
||||
key?: Prisma.SortOrder
|
||||
status?: Prisma.SortOrder
|
||||
createdAt?: Prisma.SortOrder
|
||||
updatedAt?: Prisma.SortOrder
|
||||
}
|
||||
@@ -313,9 +335,14 @@ export type StorageKeyUpdateOneWithoutAssetsNestedInput = {
|
||||
update?: Prisma.XOR<Prisma.XOR<Prisma.StorageKeyUpdateToOneWithWhereWithoutAssetsInput, Prisma.StorageKeyUpdateWithoutAssetsInput>, Prisma.StorageKeyUncheckedUpdateWithoutAssetsInput>
|
||||
}
|
||||
|
||||
export type EnumStorageKeyStatusFieldUpdateOperationsInput = {
|
||||
set?: $Enums.StorageKeyStatus
|
||||
}
|
||||
|
||||
export type StorageKeyCreateWithoutAssetsInput = {
|
||||
id?: string
|
||||
key: string
|
||||
status?: $Enums.StorageKeyStatus
|
||||
createdAt?: Date | string
|
||||
updatedAt?: Date | string
|
||||
}
|
||||
@@ -323,6 +350,7 @@ export type StorageKeyCreateWithoutAssetsInput = {
|
||||
export type StorageKeyUncheckedCreateWithoutAssetsInput = {
|
||||
id?: string
|
||||
key: string
|
||||
status?: $Enums.StorageKeyStatus
|
||||
createdAt?: Date | string
|
||||
updatedAt?: Date | string
|
||||
}
|
||||
@@ -346,6 +374,7 @@ export type StorageKeyUpdateToOneWithWhereWithoutAssetsInput = {
|
||||
export type StorageKeyUpdateWithoutAssetsInput = {
|
||||
id?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
key?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
}
|
||||
@@ -353,6 +382,7 @@ export type StorageKeyUpdateWithoutAssetsInput = {
|
||||
export type StorageKeyUncheckedUpdateWithoutAssetsInput = {
|
||||
id?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
key?: Prisma.StringFieldUpdateOperationsInput | string
|
||||
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
|
||||
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
|
||||
}
|
||||
@@ -391,6 +421,7 @@ export type StorageKeyCountOutputTypeCountAssetsArgs<ExtArgs extends runtime.Typ
|
||||
export type StorageKeySelect<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetSelect<{
|
||||
id?: boolean
|
||||
key?: boolean
|
||||
status?: boolean
|
||||
createdAt?: boolean
|
||||
updatedAt?: boolean
|
||||
assets?: boolean | Prisma.StorageKey$assetsArgs<ExtArgs>
|
||||
@@ -400,6 +431,7 @@ export type StorageKeySelect<ExtArgs extends runtime.Types.Extensions.InternalAr
|
||||
export type StorageKeySelectCreateManyAndReturn<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetSelect<{
|
||||
id?: boolean
|
||||
key?: boolean
|
||||
status?: boolean
|
||||
createdAt?: boolean
|
||||
updatedAt?: boolean
|
||||
}, ExtArgs["result"]["storageKey"]>
|
||||
@@ -407,6 +439,7 @@ export type StorageKeySelectCreateManyAndReturn<ExtArgs extends runtime.Types.Ex
|
||||
export type StorageKeySelectUpdateManyAndReturn<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetSelect<{
|
||||
id?: boolean
|
||||
key?: boolean
|
||||
status?: boolean
|
||||
createdAt?: boolean
|
||||
updatedAt?: boolean
|
||||
}, ExtArgs["result"]["storageKey"]>
|
||||
@@ -414,11 +447,12 @@ export type StorageKeySelectUpdateManyAndReturn<ExtArgs extends runtime.Types.Ex
|
||||
export type StorageKeySelectScalar = {
|
||||
id?: boolean
|
||||
key?: boolean
|
||||
status?: boolean
|
||||
createdAt?: boolean
|
||||
updatedAt?: boolean
|
||||
}
|
||||
|
||||
export type StorageKeyOmit<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetOmit<"id" | "key" | "createdAt" | "updatedAt", ExtArgs["result"]["storageKey"]>
|
||||
export type StorageKeyOmit<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetOmit<"id" | "key" | "status" | "createdAt" | "updatedAt", ExtArgs["result"]["storageKey"]>
|
||||
export type StorageKeyInclude<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = {
|
||||
assets?: boolean | Prisma.StorageKey$assetsArgs<ExtArgs>
|
||||
_count?: boolean | Prisma.StorageKeyCountOutputTypeDefaultArgs<ExtArgs>
|
||||
@@ -434,6 +468,7 @@ export type $StorageKeyPayload<ExtArgs extends runtime.Types.Extensions.Internal
|
||||
scalars: runtime.Types.Extensions.GetPayloadResult<{
|
||||
id: string
|
||||
key: string
|
||||
status: $Enums.StorageKeyStatus
|
||||
createdAt: Date
|
||||
updatedAt: Date
|
||||
}, ExtArgs["result"]["storageKey"]>
|
||||
@@ -862,6 +897,7 @@ export interface Prisma__StorageKeyClient<T, Null = never, ExtArgs extends runti
|
||||
export interface StorageKeyFieldRefs {
|
||||
readonly id: Prisma.FieldRef<"StorageKey", 'String'>
|
||||
readonly key: Prisma.FieldRef<"StorageKey", 'String'>
|
||||
readonly status: Prisma.FieldRef<"StorageKey", 'StorageKeyStatus'>
|
||||
readonly createdAt: Prisma.FieldRef<"StorageKey", 'DateTime'>
|
||||
readonly updatedAt: Prisma.FieldRef<"StorageKey", 'DateTime'>
|
||||
}
|
||||
|
||||
@@ -1311,11 +1311,7 @@ describe('AssetService', () => {
|
||||
})
|
||||
}
|
||||
|
||||
// 3. Trigger the purge job for the FIRST batch (100 items)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgePendingAssets()
|
||||
|
||||
// ...
|
||||
// 3. Trigger the purge job
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await (assetService as any).purgePendingAssets()
|
||||
|
||||
|
||||
+37
-69
@@ -18,7 +18,7 @@ import {
|
||||
UpdateAssetOrderRequest,
|
||||
UserInfo,
|
||||
} from '@/dtos/asset'
|
||||
import { Asset, AssetType, Prisma, StorageKey } from '@/generated/prisma/client.ts'
|
||||
import { Asset, AssetStatus, AssetType, Prisma, StorageKey, StorageKeyStatus } from '@/generated/prisma/client.ts'
|
||||
import { PaginatedData, paginateQuery, PaginationParams } from '@/services/pagination'
|
||||
import { s3Service } from '@/services/s3/s3'
|
||||
import { generateKeyBetween } from 'jittered-fractional-indexing'
|
||||
@@ -980,45 +980,26 @@ export class AssetService {
|
||||
|
||||
private cleanupJobsRunning = false
|
||||
|
||||
startCleanupJob() {
|
||||
async startCleanupJob() {
|
||||
this.cleanupJobsRunning = true
|
||||
|
||||
// Stage 1: Expiration Job - Move expired trashed roots to pending_purge
|
||||
const runExpiration = async () => {
|
||||
const runCleanup = async () => {
|
||||
if (!this.cleanupJobsRunning) return
|
||||
|
||||
try {
|
||||
await this.expireTrashedAssets()
|
||||
} catch (e: unknown) {
|
||||
console.error('Error in asset expiration job:', e)
|
||||
}
|
||||
setTimeout(runExpiration, 5000) // Run every 5 seconds, less frequent
|
||||
}
|
||||
|
||||
// Stage 2: Physical Purge Job - Delete files from S3 and wipe from DB
|
||||
const runPurge = async () => {
|
||||
if (!this.cleanupJobsRunning) return
|
||||
try {
|
||||
await this.purgePendingAssets()
|
||||
} catch (e: unknown) {
|
||||
console.error('Error in asset purge job:', e)
|
||||
}
|
||||
setTimeout(runPurge, 1000) // Run every 1 second
|
||||
}
|
||||
|
||||
// Stage 3: Garbage Collection Job - Purge unreferenced StorageKeys and physical files
|
||||
const runGc = async () => {
|
||||
if (!this.cleanupJobsRunning) return
|
||||
try {
|
||||
await this.purgeUnreferencedStorageKeys()
|
||||
} catch (e: unknown) {
|
||||
console.error('Error in asset GC job:', e)
|
||||
console.error('Error in asset cleanup job:', e)
|
||||
}
|
||||
|
||||
if (this.cleanupJobsRunning) {
|
||||
setTimeout(runCleanup, 5000) // Run every 5 seconds after previous run completion
|
||||
}
|
||||
setTimeout(runGc, 10000) // Run every 10 seconds
|
||||
}
|
||||
|
||||
runExpiration()
|
||||
runPurge()
|
||||
runGc()
|
||||
runCleanup()
|
||||
}
|
||||
|
||||
stopCleanupJob() {
|
||||
@@ -1062,39 +1043,19 @@ export class AssetService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Stage 2: Physically delete files and DB records for assets marked as 'pending_purge'.
|
||||
* Stage 2: Delete DB records for assets marked as 'pending_purge'.
|
||||
*/
|
||||
private async purgePendingAssets() {
|
||||
// 1. Lock 100 assets using raw SQL FOR UPDATE SKIP LOCKED and mark them as 'purging'
|
||||
// We include a timeout check (1 hour) to allow picking up stalled tasks.
|
||||
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string }[]>`
|
||||
WITH to_purge AS (
|
||||
SELECT id FROM assets
|
||||
WHERE status = 'pending_purge'
|
||||
OR (status = 'purging' AND updated_at < NOW() - INTERVAL '1 hour')
|
||||
LIMIT 100
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE assets SET status = 'purging', updated_at = NOW()
|
||||
WHERE id IN (SELECT id FROM to_purge)
|
||||
RETURNING id;
|
||||
`
|
||||
|
||||
if (lockedAssets.length === 0) return
|
||||
|
||||
const successfulIds = lockedAssets.map((a) => a.id)
|
||||
|
||||
// 3. Hard delete from database
|
||||
if (successfulIds.length > 0) {
|
||||
await this.prismaClient.asset.deleteMany({
|
||||
where: { id: { in: successfulIds } },
|
||||
})
|
||||
const { count } = await this.prismaClient.asset.deleteMany({
|
||||
where: { status: AssetStatus.pending_purge },
|
||||
})
|
||||
|
||||
if (count > 0) {
|
||||
logger.info(
|
||||
{
|
||||
purgedCount: successfulIds.length,
|
||||
purgedCount: count,
|
||||
},
|
||||
`${successfulIds.length} assets purged from database`,
|
||||
`${count} assets purged from database`,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1105,23 +1066,30 @@ export class AssetService {
|
||||
*/
|
||||
private async purgeUnreferencedStorageKeys() {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
let purgedCount = 0
|
||||
let physicalFilesDeleted = 0
|
||||
|
||||
// 1. Find storage keys with no associated assets and older than 24 hours
|
||||
// We use 24 hours to avoid race conditions with ongoing uploads or copies.
|
||||
const twentyFourHoursAgo = new Date()
|
||||
twentyFourHoursAgo.setDate(twentyFourHoursAgo.getDate() - 1)
|
||||
// using FOR UPDATE SKIP LOCKED to ensure multiple servers don't pick up the same keys.
|
||||
const lockedKeys = await this.prismaClient.$queryRaw<{ id: string; key: string }[]>`
|
||||
WITH to_purge AS (
|
||||
SELECT sk.id FROM storage_keys sk
|
||||
LEFT JOIN assets a ON a.storage_key_id = sk.id
|
||||
WHERE a.id IS NULL
|
||||
AND sk.created_at < NOW() - INTERVAL '24 hours'
|
||||
AND (sk.status = 'active' OR (sk.status = 'purging' AND sk.updated_at < NOW() - INTERVAL '1 hour'))
|
||||
LIMIT 100
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE storage_keys SET status = 'purging', updated_at = NOW()
|
||||
WHERE id IN (SELECT id FROM to_purge)
|
||||
RETURNING id, key;
|
||||
`
|
||||
|
||||
const unreferencedKeys = await this.prismaClient.storageKey.findMany({
|
||||
where: {
|
||||
assets: { none: {} },
|
||||
createdAt: { lt: twentyFourHoursAgo },
|
||||
},
|
||||
take: 100,
|
||||
})
|
||||
if (lockedKeys.length === 0) return
|
||||
|
||||
for (const sk of unreferencedKeys) {
|
||||
let physicalFilesDeleted = 0
|
||||
let purgedCount = 0
|
||||
|
||||
for (const sk of lockedKeys) {
|
||||
try {
|
||||
// Physically delete from S3
|
||||
const parts = sk.key.split('/')
|
||||
|
||||
Reference in New Issue
Block a user