feat(project): implement async project deletion and storage cleanup

This commit is contained in:
yiling
2026-05-23 18:09:30 +08:00
parent b9bb4ba143
commit 8bc275e446
9 changed files with 408 additions and 22 deletions
@@ -0,0 +1,71 @@
-- DropForeignKey
ALTER TABLE "asset_comment_attachments" DROP CONSTRAINT "asset_comment_attachments_asset_id_fkey";
-- DropForeignKey
ALTER TABLE "asset_comment_attachments" DROP CONSTRAINT "asset_comment_attachments_comment_id_fkey";
-- DropForeignKey
ALTER TABLE "asset_comments" DROP CONSTRAINT "asset_comments_asset_id_fkey";
-- DropForeignKey
ALTER TABLE "asset_comments" DROP CONSTRAINT "asset_comments_reply_to_id_fkey";
-- DropForeignKey
ALTER TABLE "assets" DROP CONSTRAINT "assets_parent_id_fkey";
-- DropForeignKey
ALTER TABLE "assets" DROP CONSTRAINT "assets_project_id_fkey";
-- DropForeignKey
ALTER TABLE "invites" DROP CONSTRAINT "invites_project_id_fkey";
-- DropForeignKey
ALTER TABLE "metadata_fields" DROP CONSTRAINT "metadata_fields_project_id_fkey";
-- DropForeignKey
ALTER TABLE "notifications" DROP CONSTRAINT "notifications_asset_id_fkey";
-- DropForeignKey
ALTER TABLE "notifications" DROP CONSTRAINT "notifications_project_id_fkey";
-- DropForeignKey
ALTER TABLE "project_members" DROP CONSTRAINT "project_members_project_id_fkey";
-- DropForeignKey
ALTER TABLE "share_links" DROP CONSTRAINT "share_links_project_id_fkey";
-- AddForeignKey
ALTER TABLE "project_members" ADD CONSTRAINT "project_members_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "share_links" ADD CONSTRAINT "share_links_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "assets" ADD CONSTRAINT "assets_parent_id_fkey" FOREIGN KEY ("parent_id") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "assets" ADD CONSTRAINT "assets_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "asset_comments" ADD CONSTRAINT "asset_comments_asset_id_fkey" FOREIGN KEY ("asset_id") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "asset_comments" ADD CONSTRAINT "asset_comments_reply_to_id_fkey" FOREIGN KEY ("reply_to_id") REFERENCES "asset_comments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "asset_comment_attachments" ADD CONSTRAINT "asset_comment_attachments_comment_id_fkey" FOREIGN KEY ("comment_id") REFERENCES "asset_comments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "asset_comment_attachments" ADD CONSTRAINT "asset_comment_attachments_asset_id_fkey" FOREIGN KEY ("asset_id") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "invites" ADD CONSTRAINT "invites_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "metadata_fields" ADD CONSTRAINT "metadata_fields_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "notifications" ADD CONSTRAINT "notifications_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "notifications" ADD CONSTRAINT "notifications_asset_id_fkey" FOREIGN KEY ("asset_id") REFERENCES "assets"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+12 -12
View File
@@ -312,7 +312,7 @@ model ProjectMember {
role ProjectMemberRole
projectId String @map("project_id")
project Project @relation("ProjectToProjectMember", fields: [projectId], references: [id])
project Project @relation("ProjectToProjectMember", fields: [projectId], references: [id], onDelete: Cascade)
teamMemberId String @map("team_member_id")
teamMember TeamMember @relation("TeamMemberToProjectMember", fields: [teamMemberId], references: [id])
@@ -334,7 +334,7 @@ model ShareLink {
updatedAt DateTime @updatedAt @map("updated_at")
projectId String @map("project_id")
project Project @relation("ProjectToShareLink", fields: [projectId], references: [id])
project Project @relation("ProjectToShareLink", fields: [projectId], references: [id], onDelete: Cascade)
rootFolderId String @unique @map("root_folder_id")
rootFolder Asset @relation("ShareLinkToRootFolder", fields: [rootFolderId], references: [id])
@@ -362,7 +362,7 @@ model Asset {
updatedAt DateTime @updatedAt @map("updated_at")
parentId String? @map("parent_id")
parent Asset? @relation("AssetChildren", fields: [parentId], references: [id])
parent Asset? @relation("AssetChildren", fields: [parentId], references: [id], onDelete: Cascade)
children Asset[] @relation("AssetChildren")
targetId String? @map("target_id")
@@ -378,7 +378,7 @@ model Asset {
task Task? @relation("TaskToAsset", fields: [taskId], references: [id])
projectId String? @map("project_id")
project Project? @relation("ProjectToAsset", fields: [projectId], references: [id])
project Project? @relation("ProjectToAsset", fields: [projectId], references: [id], onDelete: Cascade)
comments AssetComment[] @relation("AssetToComment")
commentAttachments AssetCommentAttachment[] @relation("AssetToCommentAttachment")
@@ -436,13 +436,13 @@ model AssetComment {
updatedAt DateTime @updatedAt @map("updated_at")
assetId String @map("asset_id")
asset Asset @relation("AssetToComment", fields: [assetId], references: [id])
asset Asset @relation("AssetToComment", fields: [assetId], references: [id], onDelete: Cascade)
creatorId String? @map("creator_id")
creator User? @relation("UserToAssetComment", fields: [creatorId], references: [id])
replyToId String? @map("reply_to_id")
replyTo AssetComment? @relation("CommentReplies", fields: [replyToId], references: [id])
replyTo AssetComment? @relation("CommentReplies", fields: [replyToId], references: [id], onDelete: Cascade)
replies AssetComment[] @relation("CommentReplies")
attachments AssetCommentAttachment[] @relation("CommentToAttachment")
@@ -456,10 +456,10 @@ model AssetCommentAttachment {
updatedAt DateTime @updatedAt @map("updated_at")
commentId String @map("comment_id")
comment AssetComment @relation("CommentToAttachment", fields: [commentId], references: [id])
comment AssetComment @relation("CommentToAttachment", fields: [commentId], references: [id], onDelete: Cascade)
assetId String @map("asset_id")
asset Asset @relation("AssetToCommentAttachment", fields: [assetId], references: [id])
asset Asset @relation("AssetToCommentAttachment", fields: [assetId], references: [id], onDelete: Cascade)
@@map("asset_comment_attachments")
}
@@ -490,7 +490,7 @@ model Invite {
team Team @relation("TeamToInvite", fields: [teamId], references: [id])
projectId String? @map("project_id")
project Project? @relation("ProjectToInvite", fields: [projectId], references: [id])
project Project? @relation("ProjectToInvite", fields: [projectId], references: [id], onDelete: Cascade)
inviterId String @map("inviter_id")
inviter User @relation("UserToInvite", fields: [inviterId], references: [id])
@@ -512,7 +512,7 @@ model MetadataField {
team Team? @relation("TeamToMetadataField", fields: [teamId], references: [id])
projectId String? @map("project_id")
project Project? @relation("ProjectToMetadataField", fields: [projectId], references: [id])
project Project? @relation("ProjectToMetadataField", fields: [projectId], references: [id], onDelete: Cascade)
@@map("metadata_fields")
}
@@ -527,13 +527,13 @@ model Notification {
team Team @relation("TeamToNotification", fields: [teamId], references: [id])
projectId String? @map("project_id")
project Project? @relation("ProjectToNotification", fields: [projectId], references: [id])
project Project? @relation("ProjectToNotification", fields: [projectId], references: [id], onDelete: Cascade)
creatorId String? @map("creator_id")
creator User? @relation("UserToCreatedNotification", fields: [creatorId], references: [id])
assetId String? @map("asset_id")
asset Asset? @relation("AssetToNotification", fields: [assetId], references: [id])
asset Asset? @relation("AssetToNotification", fields: [assetId], references: [id], onDelete: Cascade)
taskId String? @map("task_id")
task Task? @relation("TaskToNotification", fields: [taskId], references: [id])
+13
View File
@@ -113,6 +113,19 @@ const route = new Hono<{ Variables: { user: User } }>()
const p = await projectService.getProject(projectId)
return c.json(p)
})
.delete('/projects/:projectId', async (c) => {
const projectId = c.req.param('projectId')
const user = c.get('user')
await authzService.hasPermission({
projectId,
user,
permission: Permission.Admin,
})
await projectService.deleteProject(projectId)
return c.json({ success: true })
})
.get('/projects/:projectId/team', async (c) => {
const projectId = c.req.param('projectId')
const user = c.get('user')
+2
View File
@@ -22,6 +22,7 @@ import teamRoute from '@/api/team'
import uploadRoute from '@/api/upload'
import versionStackRoute from '@/api/versionStack'
import { metadataService } from '@/services/metadata/metadata'
import { assetService } from '@/services/asset/asset'
import { workflowService } from '@/workflow/workflow'
type User = Prisma.UserGetPayload<Record<string, never>>
@@ -61,6 +62,7 @@ export type AppType = typeof apiAppRoute
export async function run() {
// Start services
await metadataService.syncSystemFields().catch(console.error)
assetService.startCleanupJob()
workflowService.start()
if (process.env.WORKFLOW_EXECUTOR === 'temporal') {
const args = process.argv.slice(2)
+54
View File
@@ -756,6 +756,60 @@ export class AssetService {
}
}
startCleanupJob() {
// Run cleanup every 1 second
setInterval(async () => {
try {
await this.cleanupOldAssets()
} catch (e: unknown) {
console.error('Error in asset cleanup job:', e)
}
}, 1000)
}
private async cleanupOldAssets() {
const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000)
// 1. Lock 100 assets using raw SQL FOR UPDATE SKIP LOCKED and mark them as 'removed'
// This immediately releases the DB lock as soon as the status is updated.
const lockedAssets = await this.prismaClient.$queryRaw<{ id: string; key: string | null }[]>`
WITH to_delete AS (
SELECT id FROM assets
WHERE removed = true
AND deleted_at < ${thirtyDaysAgo}
AND status != 'removed'
LIMIT 100
FOR UPDATE SKIP LOCKED
)
UPDATE assets SET status = 'removed', updated_at = NOW()
WHERE id IN (SELECT id FROM to_delete)
RETURNING id, key;
`
if (lockedAssets.length === 0) return
// 2. Physical file deletion from storage
const bucket = process.env.S3_BUCKET || 'shumai'
await Promise.allSettled(
lockedAssets.map(async (a) => {
if (a.key) {
try {
await s3Service.deleteObject(bucket, a.key)
} catch (e: unknown) {
console.error(`Failed to delete storage object ${a.key}:`, e)
}
}
}),
)
// 3. Hard delete from database
// Cascade FKs will handle related comments, metadata, etc.
const ids = lockedAssets.map((a) => a.id)
await this.prismaClient.asset.deleteMany({
where: { id: { in: ids } },
})
}
async restoreAssets(ids: string[]): Promise<void> {
for (const id of ids) {
await this.prismaClient.$transaction(async (tx) => {
+138
View File
@@ -3,10 +3,12 @@ import { prisma } from '@/db'
import { setupTestDbHooks } from '@/db-test-hooks'
import { ProjectService } from './project'
import { s3Service } from '@/services/s3/s3'
import { assetService } from '@/services/asset/asset'
vi.mock('@/services/s3/s3', () => ({
s3Service: {
presign: vi.fn(),
deleteObject: vi.fn(),
},
}))
@@ -218,4 +220,140 @@ describe('ProjectService', () => {
expect(p?.shareRoot?.name).toBe('share_root')
})
})
describe('deleteProject', () => {
it('detaches and soft-deletes project assets', async () => {
const team = await prisma.team.create({ data: { name: 'Delete Team' } })
const u = await prisma.user.create({ data: { name: 'u', email: 'delete@example.com' } })
await prisma.teamMember.create({
data: { teamId: team.id, userId: u.id, role: 'owner' },
})
const project = await projectService.createProject(u, {
name: 'Project to Delete',
teamId: team.id,
})
// Add some related data
const asset = await prisma.asset.create({
data: {
name: 'Test Asset',
type: 'file',
status: 'processed',
projectId: project.id,
key: 'asset-key',
},
})
// Set cover image key
await prisma.project.update({
where: { id: project.id },
data: { coverImageKey: 'cover-key' },
})
// Delete project
await projectService.deleteProject(project.id)
// Verify project is deleted
const p = await prisma.project.findUnique({ where: { id: project.id } })
expect(p).toBeNull()
// Verify asset is soft-deleted and detached
const a = await prisma.asset.findUnique({ where: { id: asset.id } })
expect(a).toBeDefined()
expect(a?.removed).toBe(true)
expect(a?.projectId).toBeNull()
expect(a?.parentId).toBeNull()
// Verify S3 project cover cleanup (synchronous)
expect(s3Service.deleteObject).toHaveBeenCalledWith(expect.any(String), 'cover-key')
// S3 asset key cleanup is NOT called synchronously
expect(s3Service.deleteObject).not.toHaveBeenCalledWith(expect.any(String), 'asset-key')
})
it('cleans up nested hierarchy asynchronously in background job', async () => {
const team = await prisma.team.create({ data: { name: 'Nested Team' } })
const u = await prisma.user.create({ data: { name: 'u', email: 'nested@example.com' } })
await prisma.teamMember.create({
data: { teamId: team.id, userId: u.id, role: 'owner' },
})
const project = await projectService.createProject(u, {
name: 'Nested Project',
teamId: team.id,
})
const projectDb = await prisma.project.findUnique({ where: { id: project.id } })
// Create hierarchy: Folder A -> Folder B -> File C
const folderA = await prisma.asset.create({
data: {
name: 'Folder A',
type: 'folder',
status: 'processed',
projectId: project.id,
parentId: projectDb!.rootFolderId,
},
})
const folderB = await prisma.asset.create({
data: {
name: 'Folder B',
type: 'folder',
status: 'processed',
projectId: project.id,
parentId: folderA.id,
},
})
const fileC = await prisma.asset.create({
data: {
name: 'File C',
type: 'file',
status: 'processed',
projectId: project.id,
parentId: folderB.id,
key: 'file-c-key',
},
})
// Add metadata and comments to verify cascade
await prisma.assetMetadataValue.create({
data: {
assetId: fileC.id,
fieldId: 'test-field',
stringValue: 'test-value',
},
})
const comment = await prisma.assetComment.create({
data: {
assetId: fileC.id,
message: 'Test comment',
},
})
// Delete project
await projectService.deleteProject(project.id)
// Manually trigger the private cleanup method for testing
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (assetService as any).cleanupOldAssets()
// Verify everything is wiped from DB
expect(await prisma.project.findUnique({ where: { id: project.id } })).toBeNull()
expect(await prisma.asset.findUnique({ where: { id: folderA.id } })).toBeNull()
expect(await prisma.asset.findUnique({ where: { id: folderB.id } })).toBeNull()
expect(await prisma.asset.findUnique({ where: { id: fileC.id } })).toBeNull()
// Verify cascade
expect(
await prisma.assetMetadataValue.findUnique({
// eslint-disable-next-line @typescript-eslint/naming-convention
where: { assetId_fieldId: { assetId: fileC.id, fieldId: 'test-field' } },
}),
).toBeNull()
expect(await prisma.assetComment.findUnique({ where: { id: comment.id } })).toBeNull()
// Verify S3 cleanup for file
expect(s3Service.deleteObject).toHaveBeenCalledWith(expect.any(String), 'file-c-key')
})
})
})
+48 -1
View File
@@ -130,7 +130,7 @@ export class ProjectService {
const direction = req.sortDirection === 'desc' ? 'desc' : 'asc'
if (req.sortBy) {
if (req.sortBy === 'created_at') {
orderBy.id = direction
orderBy.createdAt = direction
} else if (req.sortBy === 'updated_at') {
orderBy.updatedAt = direction
} else {
@@ -224,6 +224,53 @@ export class ProjectService {
}))
}
async deleteProject(projectId: string): Promise<void> {
const project = await prisma.project.findUnique({
where: { id: projectId },
})
if (!project) throw new Error('Project not found')
const sixtyDaysAgo = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000)
await prisma.$transaction(async (tx) => {
// 1. Unlink root folders from project to avoid cycle during deletion
await tx.project.update({
where: { id: projectId },
data: { rootFolderId: null, shareRootId: null },
})
// 2. Detach ALL assets (files and folders) from project and soft-delete them
// This queues them for the background cleanup job.
await tx.asset.updateMany({
where: { projectId },
data: {
projectId: null,
parentId: null,
targetId: null,
removed: true,
deletedAt: sixtyDaysAgo,
status: 'trashed',
},
})
// 3. Delete the project itself
// Thanks to cascade deletes, project members, invites, etc. will be cleaned up.
await tx.project.delete({
where: { id: projectId },
})
})
// Synchronously delete the project's cover image if it exists
if (project.coverImageKey) {
const bucket = process.env.S3_BUCKET || 'shumai'
try {
await s3Service.deleteObject(bucket, project.coverImageKey)
} catch (e: unknown) {
console.error(`Failed to delete project cover image ${project.coverImageKey}:`, e)
}
}
}
async getProjectRootFolder(projectId: string): Promise<string> {
const project = await prisma.project.findUnique({
where: { id: projectId },
+23
View File
@@ -4,6 +4,7 @@ import {
HeadObjectCommand,
ListObjectsV2Command,
PutObjectCommand,
DeleteObjectCommand,
S3Client,
} from '@aws-sdk/client-s3'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
@@ -49,6 +50,7 @@ export interface S3Service {
): Promise<void>
getObject(bucket: string, key: string): Promise<S3Object>
downloadToFile(bucket: string, key: string, filePath: string): Promise<void>
deleteObject(bucket: string, key: string): Promise<void>
headObject(bucket: string, key: string): Promise<ObjectInfo>
listObjects(bucket: string, prefix: string): Promise<string[]>
uploadFile(filePath: string, contentType: string): Promise<string>
@@ -157,6 +159,14 @@ export class S3StorageService implements S3Service {
})
}
async deleteObject(bucket: string, key: string): Promise<void> {
const command = new DeleteObjectCommand({
Bucket: bucket,
Key: key,
})
await this.client.send(command)
}
async headObject(bucket: string, key: string): Promise<ObjectInfo> {
const command = new HeadObjectCommand({
Bucket: bucket,
@@ -311,6 +321,19 @@ export class LocalStorageService implements S3Service {
await fs.promises.copyFile(srcPath, filePath)
}
async deleteObject(bucket: string, key: string): Promise<void> {
const filePath = this.getFilePath(bucket, key)
try {
await fs.promises.unlink(filePath)
} catch (e: unknown) {
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
// Ignore if file doesn't exist
return
}
throw e
}
}
async headObject(bucket: string, key: string): Promise<ObjectInfo> {
const filePath = this.getFilePath(bucket, key)
try {
+47 -9
View File
@@ -26,14 +26,16 @@ import {
import { ShumaiLogo } from '@/ui/components/ui/icons'
import { formatDateAgo } from '@/ui/lib/time'
import { cn } from '@/ui/lib/utils'
import { useQuery, useSuspenseQuery } from '@tanstack/react-query'
import { useQuery, useSuspenseQuery, useQueryClient } from '@tanstack/react-query'
import { createFileRoute, useNavigate } from '@tanstack/react-router'
import { MoreHorizontal, PlusIcon } from 'lucide-react'
import { useState } from 'react'
import { Input } from '@/ui/components/ui/input'
function TeamPage() {
const { teamId } = Route.useParams()
const navigate = useNavigate()
const queryClient = useQueryClient()
// State for clicked project for animation
const [clickedProjectId, setClickedProjectId] = useState<string | null>(null)
@@ -46,6 +48,7 @@ function TeamPage() {
// State for Delete Alert
const [isDeleteDialogOpen, setIsDeleteDialogOpen] = useState(false)
const [projectToDelete, setProjectToDelete] = useState<ProjectInfo | undefined>(undefined)
const [deleteConfirmText, setDeleteConfirmText] = useState('')
// State for Members Dialog
const [isMembersDialogOpen, setIsMembersDialogOpen] = useState(false)
@@ -105,6 +108,18 @@ function TeamPage() {
},
})
const deleteProjectMutation = useMutation({
mutationFn: async (projectId: string) => {
const res = await client.api.projects[':projectId'].$delete({
param: { projectId },
})
if (!res.ok) throw new Error('Failed to delete project')
},
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'projects'] })
},
})
const handleCreateProjectClick = () => {
setProjectDialogMode('create')
setSelectedProject(undefined)
@@ -130,14 +145,19 @@ function TeamPage() {
const handleDeleteProjectClick = (project: ProjectInfo) => {
setProjectToDelete(project)
setDeleteConfirmText('')
setIsDeleteDialogOpen(true)
}
const handleConfirmDelete = () => {
if (projectToDelete) {
console.log('Delete project', projectToDelete.id)
const handleConfirmDelete = async () => {
if (projectToDelete && deleteConfirmText === 'delete') {
try {
await deleteProjectMutation.mutateAsync(projectToDelete.id!)
setIsDeleteDialogOpen(false)
} catch (error) {
console.error(error)
}
}
setIsDeleteDialogOpen(false)
}
const handleCardClick = (projectId: string, e: React.MouseEvent) => {
@@ -317,16 +337,34 @@ function TeamPage() {
<AlertDialogTitle>Are you absolutely sure?</AlertDialogTitle>
<AlertDialogDescription>
This action cannot be undone. This will permanently delete the project &quot;
{projectToDelete?.name}&quot;.
{projectToDelete?.name}&quot; and all its assets.
<div className="mt-4 flex flex-col gap-2 text-foreground">
<span className="text-sm font-medium">
Type <strong>delete</strong> to confirm:
</span>
<Input
value={deleteConfirmText}
onChange={(e) => setDeleteConfirmText(e.target.value)}
placeholder="delete"
className="h-9"
autoFocus
/>
</div>
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogCancel disabled={deleteProjectMutation.isPending}>
Cancel
</AlertDialogCancel>
<AlertDialogAction
onClick={handleConfirmDelete}
onClick={(e) => {
e.preventDefault()
handleConfirmDelete()
}}
disabled={deleteConfirmText !== 'delete' || deleteProjectMutation.isPending}
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
>
Delete
{deleteProjectMutation.isPending ? 'Deleting...' : 'Delete'}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>