From 688e1a4ede1b5ccef943dcf06deefabde4db87d7 Mon Sep 17 00:00:00 2001 From: yiling Date: Mon, 8 Jun 2026 12:14:14 +0800 Subject: [PATCH] feat(api,webui): implement image compression, auto-refresh for uploads, and infinite pagination for collections/shares --- AGENTS.md | 5 +- packages/agent/src/tools/create-file.ts | 2 +- packages/agent/src/tools/create-version.ts | 2 +- packages/api/src/api/file.ts | 36 +++- packages/core/src/asset/asset.test.ts | 6 +- packages/core/src/upload/upload.ts | 2 +- packages/transcode/src/transcode.ts | 8 +- .../transcode/src/workflows/transcode.test.ts | 12 +- packages/webui/components/folder-tree.tsx | 160 ++++++++++++------ packages/webui/components/upload-tasks.tsx | 3 + .../projects/$projectId/collections/index.tsx | 44 ++++- .../projects/$projectId/shares/index.tsx | 46 ++++- 12 files changed, 232 insertions(+), 94 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 403e2c95..df5c74fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -322,9 +322,8 @@ model Example { ## Infinite Scroll & Pagination -- **Frontend**: Use the `useInfiniteScroll` hook for all list UIs. - - Located in `packages/webui/hooks/use-infinite-scroll.ts`. - - Pass a `fetchData` function that accepts `page` and `limit`. +- **Frontend**: For paginated APIs (which contain `pageInfo` and `total` fields), the UI MUST use an auto infinite scroll style design using `useInfiniteQuery` from `@tanstack/react-query` and `react-intersection-observer`. Use the `total` count from the API response instead of counting the results locally. + - The default `limit` should be **20**. - **Backend**: - Most list APIs support cursor pagination using opaque tokens (via `hyrumtoken` logic) using the `paginateQuery` helper function. - Pass `PaginationParams` containing `first` (limit) and `after` (cursor). diff --git a/packages/agent/src/tools/create-file.ts b/packages/agent/src/tools/create-file.ts index 4bc2f83e..6ccd84c5 100644 --- a/packages/agent/src/tools/create-file.ts +++ b/packages/agent/src/tools/create-file.ts @@ -64,7 +64,7 @@ export function createCreateFileTool(userId: string): AgentTool() return c.json('file too large, max 10MB', 400) } - const key = `files/${ulid()}` - const buffer = Buffer.from(await file.arrayBuffer()) + let key = `files/${ulid()}` + let buffer = Buffer.from(await file.arrayBuffer()) + let contentType = file.type - await s3Service.putObject(process.env.S3_BUCKET || 'shumai', key, buffer, file.size, file.type) + if ( + file.type.startsWith('image/') && + !file.type.includes('svg') && + !file.type.includes('gif') + ) { + const tmpOut = path.join(os.tmpdir(), `transcode-${ulid()}.webp`) + try { + await transcodeService.transcodeImage(buffer, tmpOut, -1, 80, 540) + buffer = fs.readFileSync(tmpOut) + contentType = 'image/webp' + key = `${key}.webp` + } catch (err) { + console.error('Failed to compress image:', err) + // Fallback to original image if transcoding fails + } finally { + if (fs.existsSync(tmpOut)) fs.unlinkSync(tmpOut) + } + } + + await s3Service.putObject( + process.env.S3_BUCKET || 'shumai', + key, + buffer, + buffer.length, + contentType, + ) return c.json({ key }) }) diff --git a/packages/core/src/asset/asset.test.ts b/packages/core/src/asset/asset.test.ts index 25d156a1..9bb17e54 100644 --- a/packages/core/src/asset/asset.test.ts +++ b/packages/core/src/asset/asset.test.ts @@ -1446,7 +1446,7 @@ describe('AssetService', () => { expect(remainingChildren).toBe(0) }) - it('should delete the entire directory prefix for assets with complex keys (e.g., file/ULID/raw)', async () => { + it('should delete the entire directory prefix for assets with complex keys (e.g., files/ULID/raw)', async () => { const { project } = await setupBasicAssets() const { s3Service } = await import('../s3/s3') @@ -1460,7 +1460,7 @@ describe('AssetService', () => { project: { connect: { id: project.id } }, storageKey: { create: { - key: 'file/01KSBRJVY3DPK111S2MECFKDQ4/raw', + key: 'files/01KSBRJVY3DPK111S2MECFKDQ4/raw', createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000), }, }, @@ -1478,7 +1478,7 @@ describe('AssetService', () => { // Verify deletePrefix was called with the correct prefix instead of just deleteObject expect(s3Service.deletePrefix).toHaveBeenCalledWith( expect.any(String), - 'file/01KSBRJVY3DPK111S2MECFKDQ4/', + 'files/01KSBRJVY3DPK111S2MECFKDQ4/', ) // Verify record is gone diff --git a/packages/core/src/upload/upload.ts b/packages/core/src/upload/upload.ts index a825fd35..313b37f2 100644 --- a/packages/core/src/upload/upload.ts +++ b/packages/core/src/upload/upload.ts @@ -116,7 +116,7 @@ export class UploadService { const assetType = file.type === 'folder' ? AssetType.folder : AssetType.file let key: string | null = null if (assetType === AssetType.file) { - key = `file/${ulid()}/raw` + key = `files/${ulid()}/raw` } const newAsset = await this.prismaClient.asset.create({ diff --git a/packages/transcode/src/transcode.ts b/packages/transcode/src/transcode.ts index 942855dc..422e6a27 100644 --- a/packages/transcode/src/transcode.ts +++ b/packages/transcode/src/transcode.ts @@ -274,14 +274,14 @@ export class TranscodeService { } async transcodeImage( - inputFile: string, + inputFile: string | Buffer, outputFile: string, width: number, quality: number, height: number | null = null, ): Promise { let input: string | Buffer = inputFile - if (inputFile.startsWith('http')) { + if (typeof inputFile === 'string' && inputFile.startsWith('http')) { const resp = await fetch(inputFile) if (!resp.ok) { throw new Error(`Failed to fetch image from ${inputFile}: ${resp.statusText}`) @@ -512,7 +512,7 @@ export class TranscodeService { } // 6. Upload to S3 - const s3Key = `file/${params.assetId}/screenshots/${outName}` + const s3Key = `files/${params.assetId}/screenshots/${outName}` const fileBuffer = fs.readFileSync(localShotPath) await s3Service.putObject(bucket, s3Key, fileBuffer, fileBuffer.length, 'image/webp') @@ -556,7 +556,7 @@ export class TranscodeService { .toFile(localOutPath) // 5. Upload to S3 - const s3Key = `file/${params.assetId}/annotations/${outName}` + const s3Key = `files/${params.assetId}/annotations/${outName}` const fileBuffer = fs.readFileSync(localOutPath) await s3Service.putObject(bucket, s3Key, fileBuffer, fileBuffer.length, 'image/webp') diff --git a/packages/transcode/src/workflows/transcode.test.ts b/packages/transcode/src/workflows/transcode.test.ts index 4bdbe8ac..6a86490d 100644 --- a/packages/transcode/src/workflows/transcode.test.ts +++ b/packages/transcode/src/workflows/transcode.test.ts @@ -274,8 +274,8 @@ describe('Transcode Workflow', () => { }) mockActivities.takeScreenshotsActivity.mockResolvedValue([ - { key: 'file/asset-video/screenshots/shot1.webp', timestamp: 0.0 }, - { key: 'file/asset-video/screenshots/shot2.webp', timestamp: 5.0 }, + { key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 }, + { key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 }, ]) await transcodeMedia(task) @@ -295,8 +295,8 @@ describe('Transcode Workflow', () => { status: WorkflowTaskStatus.completed, output: { screenshots: [ - { key: 'file/asset-video/screenshots/shot1.webp', timestamp: 0.0 }, - { key: 'file/asset-video/screenshots/shot2.webp', timestamp: 5.0 }, + { key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 }, + { key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 }, ], }, }) @@ -336,7 +336,7 @@ describe('Transcode Workflow', () => { }) mockActivities.overlayAnnotationsActivity.mockResolvedValue( - 'file/asset-image/annotations/ann1.webp', + 'files/asset-image/annotations/ann1.webp', ) await transcodeMedia(task) @@ -350,7 +350,7 @@ describe('Transcode Workflow', () => { expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({ taskId: 'task-image-ann', status: WorkflowTaskStatus.completed, - output: { key: 'file/asset-image/annotations/ann1.webp' }, + output: { key: 'files/asset-image/annotations/ann1.webp' }, }) }) }) diff --git a/packages/webui/components/folder-tree.tsx b/packages/webui/components/folder-tree.tsx index 3db77c50..fa37c92e 100644 --- a/packages/webui/components/folder-tree.tsx +++ b/packages/webui/components/folder-tree.tsx @@ -1,44 +1,4 @@ -import type { AssetInfoPaginatedList, AssetInfo } from '@shumai/dtos' import { client } from '@/ui/api/client' -import { usePermissions } from '@/ui/hooks/use-permissions' -import { cn } from '@/ui/lib/utils' -import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useMatch, useNavigate, useParams } from '@tanstack/react-router' -import { - ChevronDown, - ChevronsDownUp, - ChevronsUpDown, - ChevronRight, - Clapperboard, - Folder, - Loader2, - Trash2, - Plus, - Share2, - LayoutGrid, - Bookmark, - MoreHorizontal, -} from 'lucide-react' -import { useEffect, useMemo, useState } from 'react' -import { useInView } from 'react-intersection-observer' -import { useDraggable, useDroppable } from '@dnd-kit/react' -import type { DragState } from './dnd-types' -import { toast } from 'sonner' -import type { ShareLinkInfo } from '@shumai/dtos' -import type { CollectionInfo } from '@shumai/dtos' -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from '@/ui/components/ui/dropdown-menu' -import { - Dialog, - DialogContent, - DialogHeader, - DialogTitle, - DialogFooter, -} from '@/ui/components/ui/dialog' import { AlertDialog, AlertDialogAction, @@ -49,8 +9,46 @@ import { AlertDialogHeader, AlertDialogTitle, } from '@/ui/components/ui/alert-dialog' -import { Input } from '@/ui/components/ui/input' import { Button } from '@/ui/components/ui/button' +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/ui/components/ui/dialog' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/ui/components/ui/dropdown-menu' +import { Input } from '@/ui/components/ui/input' +import { usePermissions } from '@/ui/hooks/use-permissions' +import { cn } from '@/ui/lib/utils' +import { useDraggable, useDroppable } from '@dnd-kit/react' +import type { AssetInfo, AssetInfoPaginatedList, CollectionInfo, ShareLinkInfo } from '@shumai/dtos' +import { useInfiniteQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { useMatch, useNavigate, useParams } from '@tanstack/react-router' +import { + Bookmark, + ChevronDown, + ChevronRight, + ChevronsDownUp, + ChevronsUpDown, + Clapperboard, + Folder, + LayoutGrid, + Loader2, + MoreHorizontal, + Plus, + Share2, + Trash2, +} from 'lucide-react' +import { useEffect, useMemo, useState } from 'react' +import { useInView } from 'react-intersection-observer' +import { toast } from 'sonner' +import type { DragState } from './dnd-types' interface FolderTreeProps { teamId: string @@ -87,32 +85,76 @@ export function FolderTree({ const [isCollectionsExpanded, setIsCollectionsExpanded] = useState(true) const [isSharesExpanded, setIsSharesExpanded] = useState(true) - const { data: shareLinksData } = useQuery({ + const { + data: shareLinksData, + fetchNextPage: fetchNextShareLinks, + hasNextPage: hasNextShareLinks, + isFetchingNextPage: isFetchingNextShareLinks, + } = useInfiniteQuery({ queryKey: ['shares', projectId], - queryFn: async () => { + queryFn: async ({ pageParam }) => { const res = await client.api.projects[':projectId'].shares.$get({ param: { projectId }, - query: { first: '100' }, + query: { first: '20', after: pageParam as string }, }) if (!res.ok) throw new Error('Failed to fetch share links') - return (await res.json()) as unknown as { data: ShareLinkInfo[] } + return (await res.json()) as unknown as { + data: ShareLinkInfo[] + pageInfo: { cursor?: string; total?: number } + } }, + initialPageParam: '', + getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined, enabled: !!projectId, }) - const { data: collectionsData } = useQuery({ + const { + data: collectionsData, + fetchNextPage: fetchNextCollections, + hasNextPage: hasNextCollections, + isFetchingNextPage: isFetchingNextCollections, + } = useInfiniteQuery({ queryKey: ['collections', projectId], - queryFn: async () => { + queryFn: async ({ pageParam }) => { const res = await client.api.projects[':projectId'].collections.$get({ param: { projectId }, - query: { first: '100' }, + query: { first: '20', after: pageParam as string }, }) if (!res.ok) throw new Error('Failed to fetch collections') - return (await res.json()) as unknown as { data: CollectionInfo[] } + return (await res.json()) as unknown as { + data: CollectionInfo[] + pageInfo: { cursor?: string; total?: number } + } }, + initialPageParam: '', + getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined, enabled: !!projectId, }) + const { ref: collectionsInViewRef, inView: collectionsInView } = useInView() + const { ref: sharesInViewRef, inView: sharesInView } = useInView() + + useEffect(() => { + if (collectionsInView && hasNextCollections && !isFetchingNextCollections) { + fetchNextCollections() + } + }, [collectionsInView, hasNextCollections, isFetchingNextCollections, fetchNextCollections]) + + useEffect(() => { + if (sharesInView && hasNextShareLinks && !isFetchingNextShareLinks) { + fetchNextShareLinks() + } + }, [sharesInView, hasNextShareLinks, isFetchingNextShareLinks, fetchNextShareLinks]) + + const collections = useMemo( + () => collectionsData?.pages.flatMap((p) => p.data) || [], + [collectionsData], + ) + const shareLinks = useMemo( + () => shareLinksData?.pages.flatMap((p) => p.data) || [], + [shareLinksData], + ) + const { mutate: createCollection } = useMutation({ mutationFn: async () => { const res = await client.api.projects[':projectId'].collections.$post({ @@ -276,8 +318,6 @@ export function FolderTree({ }, }) - const shareLinks = shareLinksData?.data ?? [] - return (
- All Collections ({collectionsData?.data.length || 0}) + All Collections ({collectionsData?.pages[0]?.pageInfo?.total || 0})
- {collectionsData?.data.map((collection) => { + {collections.map((collection) => { const isColActive = params?.collectionId === collection.id return (
) })} +
+ {isFetchingNextCollections && ( +
+ +
+ )}
)} @@ -534,7 +580,7 @@ export function FolderTree({ - All Share Links ({shareLinks.length}) + All Share Links ({shareLinksData?.pages[0]?.pageInfo?.total || 0}) @@ -558,6 +604,12 @@ export function FolderTree({ }} /> ))} +
+ {isFetchingNextShareLinks && ( +
+ +
+ )}
)} diff --git a/packages/webui/components/upload-tasks.tsx b/packages/webui/components/upload-tasks.tsx index ea25e57f..427008ec 100644 --- a/packages/webui/components/upload-tasks.tsx +++ b/packages/webui/components/upload-tasks.tsx @@ -2,6 +2,7 @@ import { client } from '@/ui/api/client' import type { TaskInfo } from '@shumai/dtos' import { useTeamId } from '@/ui/hooks/use-team-id' import { useInfiniteQuery } from '@tanstack/react-query' +import { useUploadStore } from '@/ui/stores/upload' import { Loader2, CheckCircle2, Clock, UploadCloud } from 'lucide-react' import { useEffect } from 'react' import { useInView } from 'react-intersection-observer' @@ -90,6 +91,7 @@ interface TaskGroup { export function UploadTasks() { const teamId = useTeamId() + const uploading = useUploadStore((state) => state.uploading) const { data: tasksData, fetchNextPage, @@ -111,6 +113,7 @@ export function UploadTasks() { return lastPage.pageInfo?.cursor || undefined }, enabled: !!teamId, + refetchInterval: uploading > 0 ? 2000 : false, }) const { ref, inView } = useInView() diff --git a/packages/webui/routes/projects/$projectId/collections/index.tsx b/packages/webui/routes/projects/$projectId/collections/index.tsx index 07d37224..c41fea8c 100644 --- a/packages/webui/routes/projects/$projectId/collections/index.tsx +++ b/packages/webui/routes/projects/$projectId/collections/index.tsx @@ -12,10 +12,11 @@ import { import { Avatar, AvatarImage, AvatarFallback } from '@/ui/components/ui/avatar' import { useUiStore } from '@/ui/stores/ui' import { useTeamContextStore } from '@/ui/stores/team-context' -import { useQuery, useSuspenseQuery } from '@tanstack/react-query' +import { useInfiniteQuery, useSuspenseQuery } from '@tanstack/react-query' import { createFileRoute, useNavigate } from '@tanstack/react-router' -import { useEffect, useState } from 'react' -import { Bookmark, Calendar, LayoutGrid } from 'lucide-react' +import { useEffect, useMemo, useState } from 'react' +import { Bookmark, Calendar, LayoutGrid, Loader2 } from 'lucide-react' +import { useInView } from 'react-intersection-observer' import type { CollectionInfo } from '@shumai/dtos' import { ProjectFolderSkeleton } from '@/ui/components/loading-skeletons' @@ -59,25 +60,47 @@ function CollectionsPage() { setFileListLeftSidebarCollapsed: setIsLeftSidebarCollapsed, } = useUiStore() - const { data: collectionsData, isLoading: isCollectionsLoading } = useQuery({ + const { + data: collectionsData, + isLoading: isCollectionsLoading, + fetchNextPage, + hasNextPage, + isFetchingNextPage, + } = useInfiniteQuery({ queryKey: ['collections', projectId], - queryFn: async () => { + queryFn: async ({ pageParam }) => { const res = await client.api.projects[':projectId'].collections.$get({ param: { projectId }, - query: { first: '100' }, + query: { first: '20', after: pageParam as string }, }) if (!res.ok) throw new Error('Failed to fetch collections') - return (await res.json()) as unknown as { data: CollectionInfo[] } + return (await res.json()) as unknown as { + data: CollectionInfo[] + pageInfo: { cursor?: string; total?: number } + } }, + initialPageParam: '', + getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined, enabled: !!projectId, }) + const { ref: inViewRef, inView } = useInView() + + useEffect(() => { + if (inView && hasNextPage && !isFetchingNextPage) { + fetchNextPage() + } + }, [inView, hasNextPage, isFetchingNextPage, fetchNextPage]) + + const collections = useMemo( + () => collectionsData?.pages.flatMap((p) => p.data) || [], + [collectionsData], + ) + if (!teamId || !rootFolderId || !projectInfo) { return } - const collections = collectionsData?.data ?? [] - return (
@@ -187,6 +210,9 @@ function CollectionsPage() { ))} +
+ {isFetchingNextPage && } +
)}
diff --git a/packages/webui/routes/projects/$projectId/shares/index.tsx b/packages/webui/routes/projects/$projectId/shares/index.tsx index 5ebac6a6..b74b7fc3 100644 --- a/packages/webui/routes/projects/$projectId/shares/index.tsx +++ b/packages/webui/routes/projects/$projectId/shares/index.tsx @@ -14,10 +14,16 @@ import { import { Avatar, AvatarImage, AvatarFallback } from '@/ui/components/ui/avatar' import { useUiStore } from '@/ui/stores/ui' import { useTeamContextStore } from '@/ui/stores/team-context' -import { useMutation, useQuery, useQueryClient, useSuspenseQuery } from '@tanstack/react-query' +import { + useMutation, + useInfiniteQuery, + useQueryClient, + useSuspenseQuery, +} from '@tanstack/react-query' import { createFileRoute, useNavigate } from '@tanstack/react-router' -import { useEffect, useState } from 'react' -import { Copy, ExternalLink, Calendar, Link2, Share2 } from 'lucide-react' +import { useEffect, useMemo, useState } from 'react' +import { Copy, ExternalLink, Calendar, Link2, Share2, Loader2 } from 'lucide-react' +import { useInView } from 'react-intersection-observer' import { toast } from 'sonner' import type { ShareLinkInfo } from '@shumai/dtos' import { ProjectFolderSkeleton } from '@/ui/components/loading-skeletons' @@ -63,19 +69,40 @@ function SharesPage() { setFileListLeftSidebarCollapsed: setIsLeftSidebarCollapsed, } = useUiStore() - const { data: sharesData, isLoading: isSharesLoading } = useQuery({ + const { + data: sharesData, + isLoading: isSharesLoading, + fetchNextPage, + hasNextPage, + isFetchingNextPage, + } = useInfiniteQuery({ queryKey: ['shares', projectId], - queryFn: async () => { + queryFn: async ({ pageParam }) => { const res = await client.api.projects[':projectId'].shares.$get({ param: { projectId }, - query: { first: '100' }, + query: { first: '20', after: pageParam as string }, }) if (!res.ok) throw new Error('Failed to fetch share links') - return (await res.json()) as unknown as { data: ShareLinkInfo[] } + return (await res.json()) as unknown as { + data: ShareLinkInfo[] + pageInfo: { cursor?: string; total?: number } + } }, + initialPageParam: '', + getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined, enabled: !!projectId, }) + const { ref: inViewRef, inView } = useInView() + + useEffect(() => { + if (inView && hasNextPage && !isFetchingNextPage) { + fetchNextPage() + } + }, [inView, hasNextPage, isFetchingNextPage, fetchNextPage]) + + const shares = useMemo(() => sharesData?.pages.flatMap((p) => p.data) || [], [sharesData]) + const { mutate: toggleVisibility } = useMutation({ mutationFn: async ({ id, isDisabled }: { id: string; isDisabled: boolean }) => { const res = await client.api.shares[':shareId'].$put({ @@ -98,8 +125,6 @@ function SharesPage() { return } - const shares = sharesData?.data ?? [] - const handleCopyLink = (shareId: string) => { const url = `${window.location.origin}/share/${shareId}` navigator.clipboard.writeText(url) @@ -255,6 +280,9 @@ function SharesPage() { })} +
+ {isFetchingNextPage && } +
)}