From fbff404b9bf70a1292fca39ade22db3b1117581c Mon Sep 17 00:00:00 2001 From: Yiling-J Date: Sat, 31 Oct 2026 20:11:31 +0800 Subject: [PATCH] feat(file): optimize upload and transcoding status polling with batch api (#512) * feat(file): optimize upload and transcoding status polling with batch api * fix(webui): throttle search invalidation during upload and fix stuck 100% progress ring * fix(webui): deduplicate upload search invalidation and stabilize batch polling --- CHANGELOG.md | 1 + packages/api/src/file.test.ts | 74 ++++++++ packages/api/src/file.ts | 35 ++++ packages/dtos/src/asset.ts | 5 + .../components/file-browser/file-browser.tsx | 66 +++++++- .../file-browser/file-card.test.tsx | 30 ++++ .../components/file-browser/file-card.tsx | 29 +--- .../file-browser/file-list-item.tsx | 18 +- .../file-browser/mobile-file-browser.tsx | 87 ++++++++-- .../webui/hooks/use-batch-file-status.test.ts | 158 ++++++++++++++++++ packages/webui/hooks/use-batch-file-status.ts | 96 +++++++++++ 11 files changed, 541 insertions(+), 58 deletions(-) create mode 100644 packages/webui/hooks/use-batch-file-status.test.ts create mode 100644 packages/webui/hooks/use-batch-file-status.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c3e872..cc84663c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +- **upload**: Optimize file status polling during upload and transcoding by eliminating redundant polling during active uploads and batching transcoding status checks across multiple files - **agent**: ⚠️ **Important Behavior Change**: Autofill and Embedding agents will no longer post completion comments when finishing successfully to prevent clutter in asset comment feeds. Comments are now only posted if a task encounters an error. ## [0.6.2] - 2026-10-29 diff --git a/packages/api/src/file.test.ts b/packages/api/src/file.test.ts index cbd85b5f..80870750 100644 --- a/packages/api/src/file.test.ts +++ b/packages/api/src/file.test.ts @@ -9,6 +9,7 @@ import { s3Service } from '@shumai/core/src/s3/s3' import { transcodeService } from '@shumai/core' import { authMiddleware } from './middleware/auth' import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz' +import type { AssetInfo } from '@shumai/dtos' import { renameAssetOp, resolveCommentOp, @@ -72,6 +73,7 @@ describe('file api', () => { vi.mocked(authzService.hasPermission).mockClear() vi.spyOn(assetService, 'getAsset').mockImplementation(vi.fn()) + vi.spyOn(assetService, 'listAssetsByIds').mockImplementation(vi.fn()) vi.spyOn(assetService, 'getAssetContext').mockResolvedValue({ teamId: 'test-team' }) vi.spyOn(assetService, 'updateAssetName').mockImplementation(vi.fn()) vi.spyOn(assetService, 'deleteAssets').mockImplementation(vi.fn()) @@ -126,6 +128,78 @@ describe('file api', () => { }) }) + describe('POST /files/batch', () => { + it('returns empty array when ids is empty', async () => { + const app = new Hono().use('*', authMiddleware).route('/', fileRoute) + const res = await app.request('/files/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ids: [] }), + }) + + expect(res.status).toBe(200) + const json = await res.json() + expect(json).toEqual([]) + expect(assetService.listAssetsByIds).not.toHaveBeenCalled() + }) + + it('returns asset list for accessible IDs', async () => { + const assets = [ + { id: 'f1', name: 'file-1.mp4', status: 'processing' }, + { id: 'f2', name: 'file-2.mp4', status: 'processed' }, + ] + vi.mocked(assetService.listAssetsByIds).mockResolvedValue(assets as unknown as AssetInfo[]) + + const app = new Hono().use('*', authMiddleware).route('/', fileRoute) + const res = await app.request('/files/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ids: ['f1', 'f2'] }), + }) + + expect(res.status).toBe(200) + const json = await res.json() + expect(json).toEqual(assets) + expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1', 'f2']) + }) + + it('gracefully omits unauthorized or missing assets without failing the batch', async () => { + vi.mocked(authzService.hasPermission).mockImplementation(async (req) => { + if (req.id === 'unauthorized-id') { + throw new HTTPException(403, { message: 'Permission denied' }) + } + }) + const accessibleAssets = [{ id: 'f1', name: 'file-1.mp4', status: 'processing' }] + vi.mocked(assetService.listAssetsByIds).mockResolvedValue( + accessibleAssets as unknown as AssetInfo[], + ) + + const app = new Hono().use('*', authMiddleware).route('/', fileRoute) + const res = await app.request('/files/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ids: ['f1', 'unauthorized-id'] }), + }) + + expect(res.status).toBe(200) + const json = await res.json() + expect(json).toEqual(accessibleAssets) + expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1']) + }) + + it('rejects with 400 when more than 100 IDs are requested', async () => { + const tooManyIds = Array.from({ length: 101 }, (_, i) => `id-${i}`) + const app = new Hono().use('*', authMiddleware).route('/', fileRoute) + const res = await app.request('/files/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ids: tooManyIds }), + }) + + expect(res.status).toBe(400) + }) + }) + it('PUT /files/:fileId runs rename_asset', async () => { const summary = { id: 'test-id', name: 'new-name', type: 'file', size: 10, projectId: 'p1' } vi.mocked(runOperation).mockResolvedValue(summary) diff --git a/packages/api/src/file.ts b/packages/api/src/file.ts index 67e0da48..5cdd1307 100644 --- a/packages/api/src/file.ts +++ b/packages/api/src/file.ts @@ -14,6 +14,7 @@ import { import { createCommentRequestSchema, deleteFilesRequestSchema, + batchGetFilesRequestSchema, paginationParamsSchema, restoreFilesRequestSchema, updateAssetMetadataRequestSchema, @@ -39,6 +40,40 @@ import { z } from 'zod' type User = Prisma.UserGetPayload> const route = new Hono<{ Variables: { user: User } }>() + .post('/files/batch', zValidator('json', batchGetFilesRequestSchema), async (c) => { + const user = c.get('user') + const req = c.req.valid('json') + + if (req.ids.length === 0) { + return c.json([]) + } + + const accessibleIds: string[] = [] + await Promise.all( + req.ids.map(async (id) => { + try { + await authzService.hasPermission({ + user, + permission: Permission.Read, + type: ResourceType.Asset, + id, + }) + accessibleIds.push(id) + } catch { + // Permissive: silently ignore missing or unauthorized IDs + } + }), + ) + + if (accessibleIds.length === 0) { + return c.json([]) + } + + const accessibleSet = new Set(accessibleIds) + const orderedAccessibleIds = req.ids.filter((id) => accessibleSet.has(id)) + const assets = await assetService.listAssetsByIds(orderedAccessibleIds) + return c.json(assets) + }) .get('/files/:fileId', async (c) => { const fileId = c.req.param('fileId') const user = c.get('user') diff --git a/packages/dtos/src/asset.ts b/packages/dtos/src/asset.ts index e37da2c5..89de94fb 100644 --- a/packages/dtos/src/asset.ts +++ b/packages/dtos/src/asset.ts @@ -329,6 +329,11 @@ export const deleteFilesRequestSchema = z.object({ }) export type DeleteFilesRequest = z.infer +export const batchGetFilesRequestSchema = z.object({ + ids: z.array(z.string()).max(100), +}) +export type BatchGetFilesRequest = z.infer + export const restoreFilesRequestSchema = z.object({ ids: z.array(z.string()), }) diff --git a/packages/webui/components/file-browser/file-browser.tsx b/packages/webui/components/file-browser/file-browser.tsx index dd798a0b..20a17e62 100644 --- a/packages/webui/components/file-browser/file-browser.tsx +++ b/packages/webui/components/file-browser/file-browser.tsx @@ -13,6 +13,7 @@ import type { AssetInfo, CollectionInfo, CreateUploadTaskRequest, + FileNode, SearchCondition, SearchSort, ShareLinkInfo, @@ -57,6 +58,7 @@ import { FileBrowserGridView } from './grid-view' import { FileBrowserListView } from './list-view' import { FileBrowserToolbar } from './toolbar' import { useFileActions } from './use-file-actions' +import { useBatchFileStatus } from '@/ui/hooks/use-batch-file-status' interface FileBrowserProps { teamId: string @@ -260,6 +262,8 @@ export function FileBrowser({ return [...filteredLocal, ...files] }, [files, localUploadingFiles, fileProgress]) + useBatchFileStatus({ teamId, items: displayedFiles }) + const activePreviewItem = useMemo(() => { if (selectedIds.size === 0) return null if (internalSelectedItem && selectedIds.has(internalSelectedItem.id!)) { @@ -533,6 +537,47 @@ export function FileBrowser({ }, }) + const throttledInvalidateSearch = useMemo(() => { + let timer: ReturnType | null = null + let lastRun = 0 + const THROTTLE_MS = 2500 + + const run = () => { + const now = Date.now() + if (now - lastRun >= THROTTLE_MS) { + lastRun = now + queryClient.invalidateQueries({ + queryKey: ['search', teamId, assetId], + }) + } else if (!timer) { + timer = setTimeout( + () => { + timer = null + lastRun = Date.now() + queryClient.invalidateQueries({ + queryKey: ['search', teamId, assetId], + }) + }, + THROTTLE_MS - (now - lastRun), + ) + } + } + + const cancel = () => { + if (timer) { + clearTimeout(timer) + timer = null + } + } + + const reset = () => { + cancel() + lastRun = Date.now() + } + + return Object.assign(run, { cancel, reset }) + }, [queryClient, teamId, assetId]) + const $createUploadTask = client.api.teams[':teamId'].upload.tasks.$post const { mutate: createUploadTaskMutation } = useMutation< InferResponseType, @@ -623,13 +668,19 @@ export function FileBrowser({ setLocalUploadingFiles((prev) => [...newLocalFiles, ...prev]) } - // Invalidate queries immediately so any newly created folders appear - queryClient.invalidateQueries({ - queryKey: ['search', teamId, assetId], - }) + // Invalidate queries immediately only if newly created folders appear + const hasCreatedFolders = (variables.json.files as FileNode[]).some( + (f) => f.children && f.children.length > 0, + ) + if (hasCreatedFolders) { + queryClient.invalidateQueries({ + queryKey: ['search', teamId, assetId], + }) + } queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'], }) + throttledInvalidateSearch.reset() try { await uploadFilesWithUppy({ @@ -639,14 +690,13 @@ export function FileBrowser({ storageBackend: (data.storageBackend as 's3' | 'local') || 'local', createdAssets: data.createdAssets, presignedUrls: data.presignedUrls, - onFileFinished: async (fileId) => { - await queryClient.invalidateQueries({ - queryKey: ['search', teamId, assetId], - }) + onFileFinished: (fileId) => { setLocalUploadingFiles((prev) => prev.filter((f) => f.id !== fileId)) + throttledInvalidateSearch() }, }) } finally { + throttledInvalidateSearch.cancel() if (batchIds) { setLocalUploadingFiles((prev) => prev.filter((f) => !batchIds!.has(f.id!))) } diff --git a/packages/webui/components/file-browser/file-card.test.tsx b/packages/webui/components/file-browser/file-card.test.tsx index 40977377..9da3d4b9 100644 --- a/packages/webui/components/file-browser/file-card.test.tsx +++ b/packages/webui/components/file-browser/file-card.test.tsx @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { FileCard } from './file-card' import type { AssetInfo } from '@shumai/dtos' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { useUploadStore } from '@/ui/stores/upload' vi.mock('@dnd-kit/react', () => ({ useDraggable: () => ({ @@ -341,6 +342,35 @@ describe('FileCard', () => { expect(screen.queryByText(/Alice/i)).toBeNull() }) + it('transitions from uploading to preparing circle when upload store marks file completed', () => { + useUploadStore.setState({ + fileProgress: { + 'file-uploading': { + fileId: 'file-uploading', + name: 'clip.mp4', + loaded: 1000, + total: 1000, + status: 'completed', + }, + }, + }) + + const uploadingItem: AssetInfo = { + ...fileItem, + id: 'file-uploading', + status: 'uploading', + creator: { id: 'u1', name: 'Alice' }, + } as AssetInfo + + renderComponent({ item: uploadingItem }) + + expect(screen.queryByTestId('file-card-preview-media')).toBeNull() + const circle = screen.getByTestId('file-card-preparing-circle') + expect(circle.tagName.toLowerCase()).toBe('svg') + expect(screen.getByText(/Preparing|准备中/i)).toBeTruthy() + expect(screen.queryByText(/Alice/i)).toBeNull() + }) + it('shows a preparing circle and "Preparing..." when processing without a preview yet', () => { const processingItem: AssetInfo = { ...fileItem, diff --git a/packages/webui/components/file-browser/file-card.tsx b/packages/webui/components/file-browser/file-card.tsx index ec586d18..806f86b8 100644 --- a/packages/webui/components/file-browser/file-card.tsx +++ b/packages/webui/components/file-browser/file-card.tsx @@ -119,11 +119,6 @@ export function FileCard({ : 0 : 0 - const shouldPoll = - (item.type === 'file' || item.type === 'version_stack') && - !!teamId && - (item.status === 'uploading' || item.status === 'uploaded' || item.status === 'processing') - const { data: polledItem } = useQuery({ queryKey: ['file', teamId, item.id], queryFn: async () => { @@ -133,13 +128,7 @@ export function FileCard({ if (!res.ok) throw new Error('failed to fetch file') return (await res.json()) as unknown as AssetInfo }, - enabled: shouldPoll, - refetchInterval: (query: unknown) => { - const data = (query as { state: { data: { status?: string } } }).state.data - return data?.status === 'processed' || data?.status === 'failed' || data?.status === 'error' - ? false - : 1000 - }, + enabled: false, }) const displayItem = useMemo(() => { @@ -277,16 +266,16 @@ export function FileCard({ // since the poster/sprite are generated and persisted ahead of the proxy transcodes. const hasPreview = Boolean(displayItem.preview?.thumbnailUrl || displayItem.preview?.spriteUrl) const isFailed = displayItem.status === 'failed' - const isProcessing = displayItem.status === 'processing' || displayItem.status === 'uploaded' + const isUploadDone = fileUploadState?.status === 'completed' || uploadPercent >= 100 + const isUploading = displayItem.status === 'uploading' && !isUploadDone + const isProcessing = + displayItem.status === 'processing' || + displayItem.status === 'uploaded' || + (displayItem.status === 'uploading' && isUploadDone) // While uploading/transcoding, the creator row is replaced by a short status label so the real // date/author only appears once the asset is ready or failed. - const statusText = - displayItem.status === 'uploading' - ? m.uploading() - : displayItem.status === 'uploaded' || displayItem.status === 'processing' - ? m.preparing() - : null + const statusText = isUploading ? m.uploading() : isProcessing ? m.preparing() : null const failedErrorMessage = displayItem.media?.error || m.processing_failed() @@ -393,7 +382,7 @@ export function FileCard({
- {displayItem.status === 'uploading' ? ( + {isUploading ? (
diff --git a/packages/webui/components/file-browser/file-list-item.tsx b/packages/webui/components/file-browser/file-list-item.tsx index ee5c246d..ad601799 100644 --- a/packages/webui/components/file-browser/file-list-item.tsx +++ b/packages/webui/components/file-browser/file-list-item.tsx @@ -74,11 +74,6 @@ export function FileListItem({ : 0 : 0 - const shouldPoll = - (item.type === 'file' || item.type === 'version_stack') && - !!teamId && - (item.status === 'uploading' || item.status === 'uploaded' || item.status === 'processing') - const { data: polledItem } = useQuery({ queryKey: ['file', teamId, item.id], queryFn: async () => { @@ -88,13 +83,7 @@ export function FileListItem({ if (!res.ok) throw new Error('failed to fetch file') return (await res.json()) as unknown as AssetInfo }, - enabled: shouldPoll, - refetchInterval: (query: unknown) => { - const data = (query as { state: { data: { status?: string } } }).state.data - return data?.status === 'processed' || data?.status === 'failed' || data?.status === 'error' - ? false - : 1000 - }, + enabled: false, }) const displayItem = useMemo(() => { @@ -111,6 +100,9 @@ export function FileListItem({ return polledItem || item }, [item, polledItem]) + const isUploadDone = fileUploadState?.status === 'completed' || uploadPercent >= 100 + const isUploading = displayItem.status === 'uploading' && !isUploadDone + // dnd-kit hooks const { ref: setDraggableRef, isDragging: isDraggableDragging } = useDraggable({ id: `browser:${displayItem.id!}`, @@ -220,7 +212,7 @@ export function FileListItem({ className="h-4 w-4 shrink-0 data-[state=checked]:bg-primary data-[state=checked]:border-primary" />
- {displayItem.status === 'uploading' ? ( + {isUploading ? ( ) : displayItem.status === 'failed' && !displayItem.preview?.thumbnailUrl ? ( diff --git a/packages/webui/components/file-browser/mobile-file-browser.tsx b/packages/webui/components/file-browser/mobile-file-browser.tsx index 23727afb..21eba075 100644 --- a/packages/webui/components/file-browser/mobile-file-browser.tsx +++ b/packages/webui/components/file-browser/mobile-file-browser.tsx @@ -5,7 +5,7 @@ import { usePermissions } from '@/ui/hooks/use-permissions' import { m } from '@/ui/paraglide/messages.js' import { useUploadStore } from '@/ui/stores/upload' import { uploadFilesWithUppy } from '@/ui/lib/uploader' -import type { AssetInfo, CreateUploadTaskRequest } from '@shumai/dtos' +import type { AssetInfo, CreateUploadTaskRequest, FileNode } from '@shumai/dtos' import { useMutation, useQueryClient } from '@tanstack/react-query' import type { InferRequestType, InferResponseType } from 'hono/client' import { formatSize } from '@/ui/lib/format' @@ -20,7 +20,7 @@ import { Plus, Upload, } from 'lucide-react' -import React, { useRef, useState } from 'react' +import React, { useMemo, useRef, useState } from 'react' import { useInView } from 'react-intersection-observer' import { toast } from 'sonner' import { ulid } from 'ulid' @@ -54,6 +54,7 @@ import { FileCard } from './file-card' import { FolderCard } from './folder-card' import { useFileActions } from './use-file-actions' import { MobileFileBrowserContentSkeleton } from '../loading-skeletons' +import { useBatchFileStatus } from '@/ui/hooks/use-batch-file-status' export interface MobileFileBrowserProps { teamId: string @@ -125,6 +126,7 @@ export function MobileFileBrowser({ allowDownload = true, }: MobileFileBrowserProps) { const queryClient = useQueryClient() + useBatchFileStatus({ teamId, items: files }) const { canEdit, canAdmin } = usePermissions(projectId) const fileInputRef = useRef(null) const folderInputRef = useRef(null) @@ -200,6 +202,47 @@ export function MobileFileBrowser({ // Upload actions const startTask = useUploadStore((state) => state.startTask) + const throttledInvalidateSearch = useMemo(() => { + let timer: ReturnType | null = null + let lastRun = 0 + const THROTTLE_MS = 2500 + + const run = () => { + const now = Date.now() + if (now - lastRun >= THROTTLE_MS) { + lastRun = now + queryClient.invalidateQueries({ + queryKey: ['search', teamId, assetId], + }) + } else if (!timer) { + timer = setTimeout( + () => { + timer = null + lastRun = Date.now() + queryClient.invalidateQueries({ + queryKey: ['search', teamId, assetId], + }) + }, + THROTTLE_MS - (now - lastRun), + ) + } + } + + const cancel = () => { + if (timer) { + clearTimeout(timer) + timer = null + } + } + + const reset = () => { + cancel() + lastRun = Date.now() + } + + return Object.assign(run, { cancel, reset }) + }, [queryClient, teamId, assetId]) + const $createUploadTask = client.api.teams[':teamId'].upload.tasks.$post const { mutate: createUploadTaskMutation } = useMutation< InferResponseType, @@ -218,7 +261,7 @@ export function MobileFileBrowser({ onError: () => { toast.error(m.failed_upload_file()) }, - onSuccess: async (data, _variables, context) => { + onSuccess: async (data, variables, context) => { const currentFiles = context?.files || [] const filesProgressInfo = currentFiles .map((f) => { @@ -240,22 +283,32 @@ export function MobileFileBrowser({ startTask(data.taskId, taskName, filesProgressInfo) } - queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] }) + const hasCreatedFolders = (variables.json.files as FileNode[]).some( + (f) => f.children && f.children.length > 0, + ) + if (hasCreatedFolders) { + queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] }) + } queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] }) + throttledInvalidateSearch.reset() - await uploadFilesWithUppy({ - files: currentFiles, - taskId: data.taskId!, - teamId: teamId!, - storageBackend: (data.storageBackend as 's3' | 'local') || 'local', - createdAssets: data.createdAssets, - presignedUrls: data.presignedUrls, - onFileFinished: async () => { - await queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] }) - }, - }) - queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] }) - queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] }) + try { + await uploadFilesWithUppy({ + files: currentFiles, + taskId: data.taskId!, + teamId: teamId!, + storageBackend: (data.storageBackend as 's3' | 'local') || 'local', + createdAssets: data.createdAssets, + presignedUrls: data.presignedUrls, + onFileFinished: () => { + throttledInvalidateSearch() + }, + }) + } finally { + throttledInvalidateSearch.cancel() + queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] }) + queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] }) + } }, }) diff --git a/packages/webui/hooks/use-batch-file-status.test.ts b/packages/webui/hooks/use-batch-file-status.test.ts new file mode 100644 index 00000000..84f70dd0 --- /dev/null +++ b/packages/webui/hooks/use-batch-file-status.test.ts @@ -0,0 +1,158 @@ +// @vitest-environment happy-dom +import { renderHook, waitFor } from '@testing-library/react' +import React from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import type { AssetInfo } from '@shumai/dtos' +import { useBatchFileStatus } from './use-batch-file-status' +import { client } from '@/ui/api/client' +import { useUploadStore } from '@/ui/stores/upload' + +vi.mock('@/ui/api/client', () => ({ + client: { + api: { + files: { + batch: { + $post: vi.fn(), + }, + }, + }, + }, +})) + +describe('useBatchFileStatus', () => { + let queryClient: QueryClient + + beforeEach(() => { + useUploadStore.setState({ fileProgress: {} }) + queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + }, + }) + vi.clearAllMocks() + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + const wrapper = ({ children }: { children: React.ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children) + + it('does not poll when items array is empty or no files are processing', () => { + const items: AssetInfo[] = [ + { id: 'f1', type: 'file', status: 'processed' } as AssetInfo, + { id: 'f2', type: 'file', status: 'error' } as AssetInfo, + ] + + renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper }) + + expect(client.api.files.batch.$post).not.toHaveBeenCalled() + }) + + it('does NOT poll when files are in "uploading" status and upload is in progress', () => { + const items: AssetInfo[] = [ + { id: 'f1', type: 'file', status: 'uploading' } as AssetInfo, + { id: 'f2', type: 'file', status: 'uploading' } as AssetInfo, + ] + + renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper }) + + expect(client.api.files.batch.$post).not.toHaveBeenCalled() + }) + + it('polls batch API when file has status "uploading" but upload is completed in store', async () => { + useUploadStore.setState({ + fileProgress: { + f1: { fileId: 'f1', name: 'f1.mp4', loaded: 1000, total: 1000, status: 'completed' }, + }, + }) + + const items: AssetInfo[] = [ + { id: 'f1', type: 'file', status: 'uploading' } as AssetInfo, + { id: 'f2', type: 'file', status: 'uploading' } as AssetInfo, + ] + + const updatedAssets: AssetInfo[] = [ + { id: 'f1', type: 'file', status: 'uploaded', name: 'f1.mp4' } as AssetInfo, + ] + + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + + vi.mocked(client.api.files.batch.$post).mockResolvedValue({ + ok: true, + json: async () => updatedAssets, + } as unknown as Awaited>) + + renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper }) + + await waitFor(() => { + expect(client.api.files.batch.$post).toHaveBeenCalledWith({ + json: { ids: ['f1'] }, + }) + }) + + expect(queryClient.getQueryData(['file', 'team-1', 'f1'])).toEqual(updatedAssets[0]) + // Should NOT invalidate search query for intermediate 'uploaded' state; card updates via setQueryData + expect(invalidateSpy).not.toHaveBeenCalledWith({ queryKey: ['search', 'team-1'] }) + }) + + it('polls batch API when files are in "uploaded" or "processing" status and updates cache', async () => { + const items: AssetInfo[] = [ + { id: 'f1', type: 'file', status: 'uploaded' } as AssetInfo, + { id: 'f2', type: 'file', status: 'processing' } as AssetInfo, + { id: 'f3', type: 'file', status: 'uploading' } as AssetInfo, + ] + + const updatedAssets: AssetInfo[] = [ + { id: 'f1', type: 'file', status: 'processing', name: 'f1.mp4' } as AssetInfo, + { id: 'f2', type: 'file', status: 'processed', name: 'f2.mp4' } as AssetInfo, + ] + + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + + vi.mocked(client.api.files.batch.$post).mockResolvedValue({ + ok: true, + json: async () => updatedAssets, + } as unknown as Awaited>) + + renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper }) + + await waitFor(() => { + expect(client.api.files.batch.$post).toHaveBeenCalledWith({ + json: { ids: ['f1', 'f2'] }, + }) + }) + + // Should update per-file cache in TanStack Query + expect(queryClient.getQueryData(['file', 'team-1', 'f1'])).toEqual(updatedAssets[0]) + expect(queryClient.getQueryData(['file', 'team-1', 'f2'])).toEqual(updatedAssets[1]) + + // Should invalidate folder search query because f2 finished ('processed') + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ['search', 'team-1'] }) + }) + + it('chunks requests when processing items exceed 100', async () => { + const items: AssetInfo[] = Array.from({ length: 150 }, (_, i) => ({ + id: `file-${i}`, + type: 'file', + status: 'processing', + })) as AssetInfo[] + + vi.mocked(client.api.files.batch.$post).mockResolvedValue({ + ok: true, + json: async () => [], + } as unknown as Awaited>) + + renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper }) + + await waitFor(() => { + expect(client.api.files.batch.$post).toHaveBeenCalledTimes(2) + }) + + const calls = vi.mocked(client.api.files.batch.$post).mock.calls + expect(calls[0][0].json.ids).toHaveLength(100) + expect(calls[1][0].json.ids).toHaveLength(50) + }) +}) diff --git a/packages/webui/hooks/use-batch-file-status.ts b/packages/webui/hooks/use-batch-file-status.ts new file mode 100644 index 00000000..30ef4b1f --- /dev/null +++ b/packages/webui/hooks/use-batch-file-status.ts @@ -0,0 +1,96 @@ +import { useQuery, useQueryClient } from '@tanstack/react-query' +import { useMemo, useRef } from 'react' +import type { AssetInfo } from '@shumai/dtos' +import { client } from '@/ui/api/client' +import { useUploadStore } from '@/ui/stores/upload' + +export const BATCH_SIZE = 100 +export const BATCH_POLL_INTERVAL_MS = 2000 +export const BATCH_SEARCH_INVALIDATE_THROTTLE_MS = 5000 + +export interface UseBatchFileStatusOptions { + teamId?: string + items: AssetInfo[] +} + +export function useBatchFileStatus({ teamId, items }: UseBatchFileStatusOptions): void { + const queryClient = useQueryClient() + const fileProgress = useUploadStore((state) => state.fileProgress) + const lastInvalidateTimeRef = useRef(0) + + const processingIds = useMemo(() => { + if (!teamId) return [] + return items + .filter((item) => { + if (!item.id || (item.type !== 'file' && item.type !== 'version_stack')) return false + const isUploadedOrProcessing = item.status === 'uploaded' || item.status === 'processing' + const progress = fileProgress[item.id] + const isUploadCompleted = + item.status === 'uploading' && + (progress?.status === 'completed' || + (progress?.total != null && progress.total > 0 && progress.loaded >= progress.total)) + return isUploadedOrProcessing || isUploadCompleted + }) + .map((item) => item.id!) + }, [items, teamId, fileProgress]) + + const processingIdsRef = useRef(processingIds) + processingIdsRef.current = processingIds + + const queryKey = useMemo(() => { + return ['files', 'batch', teamId] + }, [teamId]) + + useQuery({ + queryKey, + queryFn: async () => { + const currentIds = processingIdsRef.current + if (currentIds.length === 0) return [] + + const chunks: string[][] = [] + for (let i = 0; i < currentIds.length; i += BATCH_SIZE) { + chunks.push(currentIds.slice(i, i + BATCH_SIZE)) + } + + const results = await Promise.all( + chunks.map(async (chunkIds) => { + const res = await client.api.files.batch.$post({ + json: { ids: chunkIds }, + }) + if (!res.ok) throw new Error('Failed to fetch batch file statuses') + return (await res.json()) as unknown as AssetInfo[] + }), + ) + + const assets = results.flat() + + for (const asset of assets) { + if (asset.id) { + queryClient.setQueryData(['file', teamId, asset.id], asset) + } + } + + const anyFinished = assets.some( + (a) => a.status === 'processed' || a.status === 'failed' || a.status === 'error', + ) + const allFinished = + assets.length > 0 && + assets.every( + (a) => a.status === 'processed' || a.status === 'failed' || a.status === 'error', + ) + + const now = Date.now() + if ( + allFinished || + (anyFinished && now - lastInvalidateTimeRef.current >= BATCH_SEARCH_INVALIDATE_THROTTLE_MS) + ) { + lastInvalidateTimeRef.current = now + queryClient.invalidateQueries({ queryKey: ['search', teamId] }) + } + + return assets + }, + enabled: !!teamId && processingIds.length > 0, + refetchInterval: BATCH_POLL_INTERVAL_MS, + }) +}