feat(api,webui): implement image compression, auto-refresh for uploads, and infinite pagination for collections/shares

This commit is contained in:
yiling
2026-06-08 12:14:14 +08:00
parent 2ea7706d82
commit 688e1a4ede
12 changed files with 232 additions and 94 deletions
+2 -3
View File
@@ -322,9 +322,8 @@ model Example {
## Infinite Scroll & Pagination
- **Frontend**: Use the `useInfiniteScroll` hook for all list UIs.
- Located in `packages/webui/hooks/use-infinite-scroll.ts`.
- Pass a `fetchData` function that accepts `page` and `limit`.
- **Frontend**: For paginated APIs (which contain `pageInfo` and `total` fields), the UI MUST use an auto infinite scroll style design using `useInfiniteQuery` from `@tanstack/react-query` and `react-intersection-observer`. Use the `total` count from the API response instead of counting the results locally.
- The default `limit` should be **20**.
- **Backend**:
- Most list APIs support cursor pagination using opaque tokens (via `hyrumtoken` logic) using the `paginateQuery` helper function.
- Pass `PaginationParams` containing `first` (limit) and `after` (cursor).
+1 -1
View File
@@ -64,7 +64,7 @@ export function createCreateFileTool(userId: string): AgentTool<typeof createFil
const mimeType = getMimeType(absolutePath)
// Generate compliant S3 key matching normal file upload format
const s3Key = `file/${ulid()}/raw`
const s3Key = `files/${ulid()}/raw`
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
const result = await executeAgentToolWorkflow({
+1 -1
View File
@@ -65,7 +65,7 @@ export function createCreateVersionTool(userId: string): AgentTool<typeof create
const mimeType = getMimeType(absolutePath)
// Generate compliant S3 key matching normal file upload format
const s3Key = `file/${ulid()}/raw`
const s3Key = `files/${ulid()}/raw`
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
const result = await executeAgentToolWorkflow({
+33 -3
View File
@@ -6,6 +6,10 @@ import { assetService } from '@shumai/core/src/asset/asset'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { notificationService } from '@shumai/core/src/notification/notification'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/transcode/src/transcode'
import path from 'path'
import os from 'os'
import fs from 'fs'
import {
updateFileRequestSchema,
updateAssetOrderRequestSchema,
@@ -227,10 +231,36 @@ const route = new Hono<{ Variables: { user: User } }>()
return c.json('file too large, max 10MB', 400)
}
const key = `files/${ulid()}`
const buffer = Buffer.from(await file.arrayBuffer())
let key = `files/${ulid()}`
let buffer = Buffer.from(await file.arrayBuffer())
let contentType = file.type
await s3Service.putObject(process.env.S3_BUCKET || 'shumai', key, buffer, file.size, file.type)
if (
file.type.startsWith('image/') &&
!file.type.includes('svg') &&
!file.type.includes('gif')
) {
const tmpOut = path.join(os.tmpdir(), `transcode-${ulid()}.webp`)
try {
await transcodeService.transcodeImage(buffer, tmpOut, -1, 80, 540)
buffer = fs.readFileSync(tmpOut)
contentType = 'image/webp'
key = `${key}.webp`
} catch (err) {
console.error('Failed to compress image:', err)
// Fallback to original image if transcoding fails
} finally {
if (fs.existsSync(tmpOut)) fs.unlinkSync(tmpOut)
}
}
await s3Service.putObject(
process.env.S3_BUCKET || 'shumai',
key,
buffer,
buffer.length,
contentType,
)
return c.json({ key })
})
+3 -3
View File
@@ -1446,7 +1446,7 @@ describe('AssetService', () => {
expect(remainingChildren).toBe(0)
})
it('should delete the entire directory prefix for assets with complex keys (e.g., file/ULID/raw)', async () => {
it('should delete the entire directory prefix for assets with complex keys (e.g., files/ULID/raw)', async () => {
const { project } = await setupBasicAssets()
const { s3Service } = await import('../s3/s3')
@@ -1460,7 +1460,7 @@ describe('AssetService', () => {
project: { connect: { id: project.id } },
storageKey: {
create: {
key: 'file/01KSBRJVY3DPK111S2MECFKDQ4/raw',
key: 'files/01KSBRJVY3DPK111S2MECFKDQ4/raw',
createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000),
},
},
@@ -1478,7 +1478,7 @@ describe('AssetService', () => {
// Verify deletePrefix was called with the correct prefix instead of just deleteObject
expect(s3Service.deletePrefix).toHaveBeenCalledWith(
expect.any(String),
'file/01KSBRJVY3DPK111S2MECFKDQ4/',
'files/01KSBRJVY3DPK111S2MECFKDQ4/',
)
// Verify record is gone
+1 -1
View File
@@ -116,7 +116,7 @@ export class UploadService {
const assetType = file.type === 'folder' ? AssetType.folder : AssetType.file
let key: string | null = null
if (assetType === AssetType.file) {
key = `file/${ulid()}/raw`
key = `files/${ulid()}/raw`
}
const newAsset = await this.prismaClient.asset.create({
+4 -4
View File
@@ -274,14 +274,14 @@ export class TranscodeService {
}
async transcodeImage(
inputFile: string,
inputFile: string | Buffer,
outputFile: string,
width: number,
quality: number,
height: number | null = null,
): Promise<void> {
let input: string | Buffer = inputFile
if (inputFile.startsWith('http')) {
if (typeof inputFile === 'string' && inputFile.startsWith('http')) {
const resp = await fetch(inputFile)
if (!resp.ok) {
throw new Error(`Failed to fetch image from ${inputFile}: ${resp.statusText}`)
@@ -512,7 +512,7 @@ export class TranscodeService {
}
// 6. Upload to S3
const s3Key = `file/${params.assetId}/screenshots/${outName}`
const s3Key = `files/${params.assetId}/screenshots/${outName}`
const fileBuffer = fs.readFileSync(localShotPath)
await s3Service.putObject(bucket, s3Key, fileBuffer, fileBuffer.length, 'image/webp')
@@ -556,7 +556,7 @@ export class TranscodeService {
.toFile(localOutPath)
// 5. Upload to S3
const s3Key = `file/${params.assetId}/annotations/${outName}`
const s3Key = `files/${params.assetId}/annotations/${outName}`
const fileBuffer = fs.readFileSync(localOutPath)
await s3Service.putObject(bucket, s3Key, fileBuffer, fileBuffer.length, 'image/webp')
@@ -274,8 +274,8 @@ describe('Transcode Workflow', () => {
})
mockActivities.takeScreenshotsActivity.mockResolvedValue([
{ key: 'file/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'file/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
])
await transcodeMedia(task)
@@ -295,8 +295,8 @@ describe('Transcode Workflow', () => {
status: WorkflowTaskStatus.completed,
output: {
screenshots: [
{ key: 'file/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'file/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
],
},
})
@@ -336,7 +336,7 @@ describe('Transcode Workflow', () => {
})
mockActivities.overlayAnnotationsActivity.mockResolvedValue(
'file/asset-image/annotations/ann1.webp',
'files/asset-image/annotations/ann1.webp',
)
await transcodeMedia(task)
@@ -350,7 +350,7 @@ describe('Transcode Workflow', () => {
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-image-ann',
status: WorkflowTaskStatus.completed,
output: { key: 'file/asset-image/annotations/ann1.webp' },
output: { key: 'files/asset-image/annotations/ann1.webp' },
})
})
})
+106 -54
View File
@@ -1,44 +1,4 @@
import type { AssetInfoPaginatedList, AssetInfo } from '@shumai/dtos'
import { client } from '@/ui/api/client'
import { usePermissions } from '@/ui/hooks/use-permissions'
import { cn } from '@/ui/lib/utils'
import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { useMatch, useNavigate, useParams } from '@tanstack/react-router'
import {
ChevronDown,
ChevronsDownUp,
ChevronsUpDown,
ChevronRight,
Clapperboard,
Folder,
Loader2,
Trash2,
Plus,
Share2,
LayoutGrid,
Bookmark,
MoreHorizontal,
} from 'lucide-react'
import { useEffect, useMemo, useState } from 'react'
import { useInView } from 'react-intersection-observer'
import { useDraggable, useDroppable } from '@dnd-kit/react'
import type { DragState } from './dnd-types'
import { toast } from 'sonner'
import type { ShareLinkInfo } from '@shumai/dtos'
import type { CollectionInfo } from '@shumai/dtos'
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from '@/ui/components/ui/dropdown-menu'
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogFooter,
} from '@/ui/components/ui/dialog'
import {
AlertDialog,
AlertDialogAction,
@@ -49,8 +9,46 @@ import {
AlertDialogHeader,
AlertDialogTitle,
} from '@/ui/components/ui/alert-dialog'
import { Input } from '@/ui/components/ui/input'
import { Button } from '@/ui/components/ui/button'
import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/ui/components/ui/dialog'
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from '@/ui/components/ui/dropdown-menu'
import { Input } from '@/ui/components/ui/input'
import { usePermissions } from '@/ui/hooks/use-permissions'
import { cn } from '@/ui/lib/utils'
import { useDraggable, useDroppable } from '@dnd-kit/react'
import type { AssetInfo, AssetInfoPaginatedList, CollectionInfo, ShareLinkInfo } from '@shumai/dtos'
import { useInfiniteQuery, useMutation, useQueryClient } from '@tanstack/react-query'
import { useMatch, useNavigate, useParams } from '@tanstack/react-router'
import {
Bookmark,
ChevronDown,
ChevronRight,
ChevronsDownUp,
ChevronsUpDown,
Clapperboard,
Folder,
LayoutGrid,
Loader2,
MoreHorizontal,
Plus,
Share2,
Trash2,
} from 'lucide-react'
import { useEffect, useMemo, useState } from 'react'
import { useInView } from 'react-intersection-observer'
import { toast } from 'sonner'
import type { DragState } from './dnd-types'
interface FolderTreeProps {
teamId: string
@@ -87,32 +85,76 @@ export function FolderTree({
const [isCollectionsExpanded, setIsCollectionsExpanded] = useState(true)
const [isSharesExpanded, setIsSharesExpanded] = useState(true)
const { data: shareLinksData } = useQuery({
const {
data: shareLinksData,
fetchNextPage: fetchNextShareLinks,
hasNextPage: hasNextShareLinks,
isFetchingNextPage: isFetchingNextShareLinks,
} = useInfiniteQuery({
queryKey: ['shares', projectId],
queryFn: async () => {
queryFn: async ({ pageParam }) => {
const res = await client.api.projects[':projectId'].shares.$get({
param: { projectId },
query: { first: '100' },
query: { first: '20', after: pageParam as string },
})
if (!res.ok) throw new Error('Failed to fetch share links')
return (await res.json()) as unknown as { data: ShareLinkInfo[] }
return (await res.json()) as unknown as {
data: ShareLinkInfo[]
pageInfo: { cursor?: string; total?: number }
}
},
initialPageParam: '',
getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined,
enabled: !!projectId,
})
const { data: collectionsData } = useQuery({
const {
data: collectionsData,
fetchNextPage: fetchNextCollections,
hasNextPage: hasNextCollections,
isFetchingNextPage: isFetchingNextCollections,
} = useInfiniteQuery({
queryKey: ['collections', projectId],
queryFn: async () => {
queryFn: async ({ pageParam }) => {
const res = await client.api.projects[':projectId'].collections.$get({
param: { projectId },
query: { first: '100' },
query: { first: '20', after: pageParam as string },
})
if (!res.ok) throw new Error('Failed to fetch collections')
return (await res.json()) as unknown as { data: CollectionInfo[] }
return (await res.json()) as unknown as {
data: CollectionInfo[]
pageInfo: { cursor?: string; total?: number }
}
},
initialPageParam: '',
getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined,
enabled: !!projectId,
})
const { ref: collectionsInViewRef, inView: collectionsInView } = useInView()
const { ref: sharesInViewRef, inView: sharesInView } = useInView()
useEffect(() => {
if (collectionsInView && hasNextCollections && !isFetchingNextCollections) {
fetchNextCollections()
}
}, [collectionsInView, hasNextCollections, isFetchingNextCollections, fetchNextCollections])
useEffect(() => {
if (sharesInView && hasNextShareLinks && !isFetchingNextShareLinks) {
fetchNextShareLinks()
}
}, [sharesInView, hasNextShareLinks, isFetchingNextShareLinks, fetchNextShareLinks])
const collections = useMemo(
() => collectionsData?.pages.flatMap((p) => p.data) || [],
[collectionsData],
)
const shareLinks = useMemo(
() => shareLinksData?.pages.flatMap((p) => p.data) || [],
[shareLinksData],
)
const { mutate: createCollection } = useMutation({
mutationFn: async () => {
const res = await client.api.projects[':projectId'].collections.$post({
@@ -276,8 +318,6 @@ export function FolderTree({
},
})
const shareLinks = shareLinksData?.data ?? []
return (
<div className="flex flex-col h-full overflow-hidden p-3 gap-4 bg-background">
<div
@@ -406,11 +446,11 @@ export function FolderTree({
<LayoutGrid className="h-4 w-4 text-sidebar-primary" />
</div>
<span className="flex-1 truncate text-sidebar-foreground">
All Collections ({collectionsData?.data.length || 0})
All Collections ({collectionsData?.pages[0]?.pageInfo?.total || 0})
</span>
</div>
{collectionsData?.data.map((collection) => {
{collections.map((collection) => {
const isColActive = params?.collectionId === collection.id
return (
<div
@@ -478,6 +518,12 @@ export function FolderTree({
</div>
)
})}
<div ref={collectionsInViewRef} className="h-1" />
{isFetchingNextCollections && (
<div className="flex justify-center p-2">
<Loader2 className="h-4 w-4 animate-spin text-muted-foreground" />
</div>
)}
</div>
</div>
)}
@@ -534,7 +580,7 @@ export function FolderTree({
<LayoutGrid className="h-4 w-4 text-sidebar-primary" />
</div>
<span className="flex-1 truncate text-sidebar-foreground">
All Share Links ({shareLinks.length})
All Share Links ({shareLinksData?.pages[0]?.pageInfo?.total || 0})
</span>
</div>
@@ -558,6 +604,12 @@ export function FolderTree({
}}
/>
))}
<div ref={sharesInViewRef} className="h-1" />
{isFetchingNextShareLinks && (
<div className="flex justify-center p-2">
<Loader2 className="h-4 w-4 animate-spin text-muted-foreground" />
</div>
)}
</div>
</div>
)}
@@ -2,6 +2,7 @@ import { client } from '@/ui/api/client'
import type { TaskInfo } from '@shumai/dtos'
import { useTeamId } from '@/ui/hooks/use-team-id'
import { useInfiniteQuery } from '@tanstack/react-query'
import { useUploadStore } from '@/ui/stores/upload'
import { Loader2, CheckCircle2, Clock, UploadCloud } from 'lucide-react'
import { useEffect } from 'react'
import { useInView } from 'react-intersection-observer'
@@ -90,6 +91,7 @@ interface TaskGroup {
export function UploadTasks() {
const teamId = useTeamId()
const uploading = useUploadStore((state) => state.uploading)
const {
data: tasksData,
fetchNextPage,
@@ -111,6 +113,7 @@ export function UploadTasks() {
return lastPage.pageInfo?.cursor || undefined
},
enabled: !!teamId,
refetchInterval: uploading > 0 ? 2000 : false,
})
const { ref, inView } = useInView()
@@ -12,10 +12,11 @@ import {
import { Avatar, AvatarImage, AvatarFallback } from '@/ui/components/ui/avatar'
import { useUiStore } from '@/ui/stores/ui'
import { useTeamContextStore } from '@/ui/stores/team-context'
import { useQuery, useSuspenseQuery } from '@tanstack/react-query'
import { useInfiniteQuery, useSuspenseQuery } from '@tanstack/react-query'
import { createFileRoute, useNavigate } from '@tanstack/react-router'
import { useEffect, useState } from 'react'
import { Bookmark, Calendar, LayoutGrid } from 'lucide-react'
import { useEffect, useMemo, useState } from 'react'
import { Bookmark, Calendar, LayoutGrid, Loader2 } from 'lucide-react'
import { useInView } from 'react-intersection-observer'
import type { CollectionInfo } from '@shumai/dtos'
import { ProjectFolderSkeleton } from '@/ui/components/loading-skeletons'
@@ -59,25 +60,47 @@ function CollectionsPage() {
setFileListLeftSidebarCollapsed: setIsLeftSidebarCollapsed,
} = useUiStore()
const { data: collectionsData, isLoading: isCollectionsLoading } = useQuery({
const {
data: collectionsData,
isLoading: isCollectionsLoading,
fetchNextPage,
hasNextPage,
isFetchingNextPage,
} = useInfiniteQuery({
queryKey: ['collections', projectId],
queryFn: async () => {
queryFn: async ({ pageParam }) => {
const res = await client.api.projects[':projectId'].collections.$get({
param: { projectId },
query: { first: '100' },
query: { first: '20', after: pageParam as string },
})
if (!res.ok) throw new Error('Failed to fetch collections')
return (await res.json()) as unknown as { data: CollectionInfo[] }
return (await res.json()) as unknown as {
data: CollectionInfo[]
pageInfo: { cursor?: string; total?: number }
}
},
initialPageParam: '',
getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined,
enabled: !!projectId,
})
const { ref: inViewRef, inView } = useInView()
useEffect(() => {
if (inView && hasNextPage && !isFetchingNextPage) {
fetchNextPage()
}
}, [inView, hasNextPage, isFetchingNextPage, fetchNextPage])
const collections = useMemo(
() => collectionsData?.pages.flatMap((p) => p.data) || [],
[collectionsData],
)
if (!teamId || !rootFolderId || !projectInfo) {
return <ProjectFolderSkeleton />
}
const collections = collectionsData?.data ?? []
return (
<div className="flex flex-1 flex-col bg-background min-h-0">
<div className="flex flex-1 overflow-hidden relative">
@@ -187,6 +210,9 @@ function CollectionsPage() {
))}
</TableBody>
</Table>
<div ref={inViewRef} className="h-10 w-full flex items-center justify-center">
{isFetchingNextPage && <Loader2 className="h-6 w-6 animate-spin text-primary" />}
</div>
</div>
)}
</div>
@@ -14,10 +14,16 @@ import {
import { Avatar, AvatarImage, AvatarFallback } from '@/ui/components/ui/avatar'
import { useUiStore } from '@/ui/stores/ui'
import { useTeamContextStore } from '@/ui/stores/team-context'
import { useMutation, useQuery, useQueryClient, useSuspenseQuery } from '@tanstack/react-query'
import {
useMutation,
useInfiniteQuery,
useQueryClient,
useSuspenseQuery,
} from '@tanstack/react-query'
import { createFileRoute, useNavigate } from '@tanstack/react-router'
import { useEffect, useState } from 'react'
import { Copy, ExternalLink, Calendar, Link2, Share2 } from 'lucide-react'
import { useEffect, useMemo, useState } from 'react'
import { Copy, ExternalLink, Calendar, Link2, Share2, Loader2 } from 'lucide-react'
import { useInView } from 'react-intersection-observer'
import { toast } from 'sonner'
import type { ShareLinkInfo } from '@shumai/dtos'
import { ProjectFolderSkeleton } from '@/ui/components/loading-skeletons'
@@ -63,19 +69,40 @@ function SharesPage() {
setFileListLeftSidebarCollapsed: setIsLeftSidebarCollapsed,
} = useUiStore()
const { data: sharesData, isLoading: isSharesLoading } = useQuery({
const {
data: sharesData,
isLoading: isSharesLoading,
fetchNextPage,
hasNextPage,
isFetchingNextPage,
} = useInfiniteQuery({
queryKey: ['shares', projectId],
queryFn: async () => {
queryFn: async ({ pageParam }) => {
const res = await client.api.projects[':projectId'].shares.$get({
param: { projectId },
query: { first: '100' },
query: { first: '20', after: pageParam as string },
})
if (!res.ok) throw new Error('Failed to fetch share links')
return (await res.json()) as unknown as { data: ShareLinkInfo[] }
return (await res.json()) as unknown as {
data: ShareLinkInfo[]
pageInfo: { cursor?: string; total?: number }
}
},
initialPageParam: '',
getNextPageParam: (lastPage) => lastPage.pageInfo?.cursor || undefined,
enabled: !!projectId,
})
const { ref: inViewRef, inView } = useInView()
useEffect(() => {
if (inView && hasNextPage && !isFetchingNextPage) {
fetchNextPage()
}
}, [inView, hasNextPage, isFetchingNextPage, fetchNextPage])
const shares = useMemo(() => sharesData?.pages.flatMap((p) => p.data) || [], [sharesData])
const { mutate: toggleVisibility } = useMutation({
mutationFn: async ({ id, isDisabled }: { id: string; isDisabled: boolean }) => {
const res = await client.api.shares[':shareId'].$put({
@@ -98,8 +125,6 @@ function SharesPage() {
return <ProjectFolderSkeleton />
}
const shares = sharesData?.data ?? []
const handleCopyLink = (shareId: string) => {
const url = `${window.location.origin}/share/${shareId}`
navigator.clipboard.writeText(url)
@@ -255,6 +280,9 @@ function SharesPage() {
})}
</TableBody>
</Table>
<div ref={inViewRef} className="h-10 w-full flex items-center justify-center">
{isFetchingNextPage && <Loader2 className="h-6 w-6 animate-spin text-primary" />}
</div>
</div>
)}
</div>