feat(file): optimize upload and transcoding status polling with batch api (#512)

* feat(file): optimize upload and transcoding status polling with batch api

* fix(webui): throttle search invalidation during upload and fix stuck 100% progress ring

* fix(webui): deduplicate upload search invalidation and stabilize batch polling
This commit is contained in:
Yiling-J
2026-10-31 20:11:31 +08:00
committed by GitHub
parent c2348b0e68
commit fbff404b9b
11 changed files with 541 additions and 58 deletions
+1
View File
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Changed
- **upload**: Optimize file status polling during upload and transcoding by eliminating redundant polling during active uploads and batching transcoding status checks across multiple files
- **agent**: ⚠️ **Important Behavior Change**: Autofill and Embedding agents will no longer post completion comments when finishing successfully to prevent clutter in asset comment feeds. Comments are now only posted if a task encounters an error.
## [0.6.2] - 2026-10-29
+74
View File
@@ -9,6 +9,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
import { authMiddleware } from './middleware/auth'
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
import type { AssetInfo } from '@shumai/dtos'
import {
renameAssetOp,
resolveCommentOp,
@@ -72,6 +73,7 @@ describe('file api', () => {
vi.mocked(authzService.hasPermission).mockClear()
vi.spyOn(assetService, 'getAsset').mockImplementation(vi.fn())
vi.spyOn(assetService, 'listAssetsByIds').mockImplementation(vi.fn())
vi.spyOn(assetService, 'getAssetContext').mockResolvedValue({ teamId: 'test-team' })
vi.spyOn(assetService, 'updateAssetName').mockImplementation(vi.fn())
vi.spyOn(assetService, 'deleteAssets').mockImplementation(vi.fn())
@@ -126,6 +128,78 @@ describe('file api', () => {
})
})
describe('POST /files/batch', () => {
it('returns empty array when ids is empty', async () => {
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: [] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json).toEqual([])
expect(assetService.listAssetsByIds).not.toHaveBeenCalled()
})
it('returns asset list for accessible IDs', async () => {
const assets = [
{ id: 'f1', name: 'file-1.mp4', status: 'processing' },
{ id: 'f2', name: 'file-2.mp4', status: 'processed' },
]
vi.mocked(assetService.listAssetsByIds).mockResolvedValue(assets as unknown as AssetInfo[])
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: ['f1', 'f2'] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json).toEqual(assets)
expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1', 'f2'])
})
it('gracefully omits unauthorized or missing assets without failing the batch', async () => {
vi.mocked(authzService.hasPermission).mockImplementation(async (req) => {
if (req.id === 'unauthorized-id') {
throw new HTTPException(403, { message: 'Permission denied' })
}
})
const accessibleAssets = [{ id: 'f1', name: 'file-1.mp4', status: 'processing' }]
vi.mocked(assetService.listAssetsByIds).mockResolvedValue(
accessibleAssets as unknown as AssetInfo[],
)
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: ['f1', 'unauthorized-id'] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json).toEqual(accessibleAssets)
expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1'])
})
it('rejects with 400 when more than 100 IDs are requested', async () => {
const tooManyIds = Array.from({ length: 101 }, (_, i) => `id-${i}`)
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: tooManyIds }),
})
expect(res.status).toBe(400)
})
})
it('PUT /files/:fileId runs rename_asset', async () => {
const summary = { id: 'test-id', name: 'new-name', type: 'file', size: 10, projectId: 'p1' }
vi.mocked(runOperation).mockResolvedValue(summary)
+35
View File
@@ -14,6 +14,7 @@ import {
import {
createCommentRequestSchema,
deleteFilesRequestSchema,
batchGetFilesRequestSchema,
paginationParamsSchema,
restoreFilesRequestSchema,
updateAssetMetadataRequestSchema,
@@ -39,6 +40,40 @@ import { z } from 'zod'
type User = Prisma.UserGetPayload<Record<string, never>>
const route = new Hono<{ Variables: { user: User } }>()
.post('/files/batch', zValidator('json', batchGetFilesRequestSchema), async (c) => {
const user = c.get('user')
const req = c.req.valid('json')
if (req.ids.length === 0) {
return c.json([])
}
const accessibleIds: string[] = []
await Promise.all(
req.ids.map(async (id) => {
try {
await authzService.hasPermission({
user,
permission: Permission.Read,
type: ResourceType.Asset,
id,
})
accessibleIds.push(id)
} catch {
// Permissive: silently ignore missing or unauthorized IDs
}
}),
)
if (accessibleIds.length === 0) {
return c.json([])
}
const accessibleSet = new Set(accessibleIds)
const orderedAccessibleIds = req.ids.filter((id) => accessibleSet.has(id))
const assets = await assetService.listAssetsByIds(orderedAccessibleIds)
return c.json(assets)
})
.get('/files/:fileId', async (c) => {
const fileId = c.req.param('fileId')
const user = c.get('user')
+5
View File
@@ -329,6 +329,11 @@ export const deleteFilesRequestSchema = z.object({
})
export type DeleteFilesRequest = z.infer<typeof deleteFilesRequestSchema>
export const batchGetFilesRequestSchema = z.object({
ids: z.array(z.string()).max(100),
})
export type BatchGetFilesRequest = z.infer<typeof batchGetFilesRequestSchema>
export const restoreFilesRequestSchema = z.object({
ids: z.array(z.string()),
})
@@ -13,6 +13,7 @@ import type {
AssetInfo,
CollectionInfo,
CreateUploadTaskRequest,
FileNode,
SearchCondition,
SearchSort,
ShareLinkInfo,
@@ -57,6 +58,7 @@ import { FileBrowserGridView } from './grid-view'
import { FileBrowserListView } from './list-view'
import { FileBrowserToolbar } from './toolbar'
import { useFileActions } from './use-file-actions'
import { useBatchFileStatus } from '@/ui/hooks/use-batch-file-status'
interface FileBrowserProps {
teamId: string
@@ -260,6 +262,8 @@ export function FileBrowser({
return [...filteredLocal, ...files]
}, [files, localUploadingFiles, fileProgress])
useBatchFileStatus({ teamId, items: displayedFiles })
const activePreviewItem = useMemo(() => {
if (selectedIds.size === 0) return null
if (internalSelectedItem && selectedIds.has(internalSelectedItem.id!)) {
@@ -533,6 +537,47 @@ export function FileBrowser({
},
})
const throttledInvalidateSearch = useMemo(() => {
let timer: ReturnType<typeof setTimeout> | null = null
let lastRun = 0
const THROTTLE_MS = 2500
const run = () => {
const now = Date.now()
if (now - lastRun >= THROTTLE_MS) {
lastRun = now
queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
} else if (!timer) {
timer = setTimeout(
() => {
timer = null
lastRun = Date.now()
queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
},
THROTTLE_MS - (now - lastRun),
)
}
}
const cancel = () => {
if (timer) {
clearTimeout(timer)
timer = null
}
}
const reset = () => {
cancel()
lastRun = Date.now()
}
return Object.assign(run, { cancel, reset })
}, [queryClient, teamId, assetId])
const $createUploadTask = client.api.teams[':teamId'].upload.tasks.$post
const { mutate: createUploadTaskMutation } = useMutation<
InferResponseType<typeof $createUploadTask>,
@@ -623,13 +668,19 @@ export function FileBrowser({
setLocalUploadingFiles((prev) => [...newLocalFiles, ...prev])
}
// Invalidate queries immediately so any newly created folders appear
queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
// Invalidate queries immediately only if newly created folders appear
const hasCreatedFolders = (variables.json.files as FileNode[]).some(
(f) => f.children && f.children.length > 0,
)
if (hasCreatedFolders) {
queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
}
queryClient.invalidateQueries({
queryKey: ['teams', teamId, 'upload', 'tasks'],
})
throttledInvalidateSearch.reset()
try {
await uploadFilesWithUppy({
@@ -639,14 +690,13 @@ export function FileBrowser({
storageBackend: (data.storageBackend as 's3' | 'local') || 'local',
createdAssets: data.createdAssets,
presignedUrls: data.presignedUrls,
onFileFinished: async (fileId) => {
await queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
onFileFinished: (fileId) => {
setLocalUploadingFiles((prev) => prev.filter((f) => f.id !== fileId))
throttledInvalidateSearch()
},
})
} finally {
throttledInvalidateSearch.cancel()
if (batchIds) {
setLocalUploadingFiles((prev) => prev.filter((f) => !batchIds!.has(f.id!)))
}
@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { FileCard } from './file-card'
import type { AssetInfo } from '@shumai/dtos'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { useUploadStore } from '@/ui/stores/upload'
vi.mock('@dnd-kit/react', () => ({
useDraggable: () => ({
@@ -341,6 +342,35 @@ describe('FileCard', () => {
expect(screen.queryByText(/Alice/i)).toBeNull()
})
it('transitions from uploading to preparing circle when upload store marks file completed', () => {
useUploadStore.setState({
fileProgress: {
'file-uploading': {
fileId: 'file-uploading',
name: 'clip.mp4',
loaded: 1000,
total: 1000,
status: 'completed',
},
},
})
const uploadingItem: AssetInfo = {
...fileItem,
id: 'file-uploading',
status: 'uploading',
creator: { id: 'u1', name: 'Alice' },
} as AssetInfo
renderComponent({ item: uploadingItem })
expect(screen.queryByTestId('file-card-preview-media')).toBeNull()
const circle = screen.getByTestId('file-card-preparing-circle')
expect(circle.tagName.toLowerCase()).toBe('svg')
expect(screen.getByText(/Preparing|准备中/i)).toBeTruthy()
expect(screen.queryByText(/Alice/i)).toBeNull()
})
it('shows a preparing circle and "Preparing..." when processing without a preview yet', () => {
const processingItem: AssetInfo = {
...fileItem,
@@ -119,11 +119,6 @@ export function FileCard({
: 0
: 0
const shouldPoll =
(item.type === 'file' || item.type === 'version_stack') &&
!!teamId &&
(item.status === 'uploading' || item.status === 'uploaded' || item.status === 'processing')
const { data: polledItem } = useQuery({
queryKey: ['file', teamId, item.id],
queryFn: async () => {
@@ -133,13 +128,7 @@ export function FileCard({
if (!res.ok) throw new Error('failed to fetch file')
return (await res.json()) as unknown as AssetInfo
},
enabled: shouldPoll,
refetchInterval: (query: unknown) => {
const data = (query as { state: { data: { status?: string } } }).state.data
return data?.status === 'processed' || data?.status === 'failed' || data?.status === 'error'
? false
: 1000
},
enabled: false,
})
const displayItem = useMemo(() => {
@@ -277,16 +266,16 @@ export function FileCard({
// since the poster/sprite are generated and persisted ahead of the proxy transcodes.
const hasPreview = Boolean(displayItem.preview?.thumbnailUrl || displayItem.preview?.spriteUrl)
const isFailed = displayItem.status === 'failed'
const isProcessing = displayItem.status === 'processing' || displayItem.status === 'uploaded'
const isUploadDone = fileUploadState?.status === 'completed' || uploadPercent >= 100
const isUploading = displayItem.status === 'uploading' && !isUploadDone
const isProcessing =
displayItem.status === 'processing' ||
displayItem.status === 'uploaded' ||
(displayItem.status === 'uploading' && isUploadDone)
// While uploading/transcoding, the creator row is replaced by a short status label so the real
// date/author only appears once the asset is ready or failed.
const statusText =
displayItem.status === 'uploading'
? m.uploading()
: displayItem.status === 'uploaded' || displayItem.status === 'processing'
? m.preparing()
: null
const statusText = isUploading ? m.uploading() : isProcessing ? m.preparing() : null
const failedErrorMessage = displayItem.media?.error || m.processing_failed()
@@ -393,7 +382,7 @@ export function FileCard({
</div>
<div className="relative aspect-square overflow-hidden bg-muted/30">
{displayItem.status === 'uploading' ? (
{isUploading ? (
<div className="flex h-full w-full items-center justify-center bg-background/50">
<ProgressCircle progress={uploadPercent} className="w-16 h-16 z-10" />
</div>
@@ -74,11 +74,6 @@ export function FileListItem({
: 0
: 0
const shouldPoll =
(item.type === 'file' || item.type === 'version_stack') &&
!!teamId &&
(item.status === 'uploading' || item.status === 'uploaded' || item.status === 'processing')
const { data: polledItem } = useQuery({
queryKey: ['file', teamId, item.id],
queryFn: async () => {
@@ -88,13 +83,7 @@ export function FileListItem({
if (!res.ok) throw new Error('failed to fetch file')
return (await res.json()) as unknown as AssetInfo
},
enabled: shouldPoll,
refetchInterval: (query: unknown) => {
const data = (query as { state: { data: { status?: string } } }).state.data
return data?.status === 'processed' || data?.status === 'failed' || data?.status === 'error'
? false
: 1000
},
enabled: false,
})
const displayItem = useMemo(() => {
@@ -111,6 +100,9 @@ export function FileListItem({
return polledItem || item
}, [item, polledItem])
const isUploadDone = fileUploadState?.status === 'completed' || uploadPercent >= 100
const isUploading = displayItem.status === 'uploading' && !isUploadDone
// dnd-kit hooks
const { ref: setDraggableRef, isDragging: isDraggableDragging } = useDraggable({
id: `browser:${displayItem.id!}`,
@@ -220,7 +212,7 @@ export function FileListItem({
className="h-4 w-4 shrink-0 data-[state=checked]:bg-primary data-[state=checked]:border-primary"
/>
<div className="w-8 h-8 shrink-0 flex items-center justify-center bg-muted rounded overflow-hidden">
{displayItem.status === 'uploading' ? (
{isUploading ? (
<ProgressCircle progress={uploadPercent} className="w-7 h-7" />
) : displayItem.status === 'failed' && !displayItem.preview?.thumbnailUrl ? (
<AlertCircle className="h-4 w-4 text-destructive" />
@@ -5,7 +5,7 @@ import { usePermissions } from '@/ui/hooks/use-permissions'
import { m } from '@/ui/paraglide/messages.js'
import { useUploadStore } from '@/ui/stores/upload'
import { uploadFilesWithUppy } from '@/ui/lib/uploader'
import type { AssetInfo, CreateUploadTaskRequest } from '@shumai/dtos'
import type { AssetInfo, CreateUploadTaskRequest, FileNode } from '@shumai/dtos'
import { useMutation, useQueryClient } from '@tanstack/react-query'
import type { InferRequestType, InferResponseType } from 'hono/client'
import { formatSize } from '@/ui/lib/format'
@@ -20,7 +20,7 @@ import {
Plus,
Upload,
} from 'lucide-react'
import React, { useRef, useState } from 'react'
import React, { useMemo, useRef, useState } from 'react'
import { useInView } from 'react-intersection-observer'
import { toast } from 'sonner'
import { ulid } from 'ulid'
@@ -54,6 +54,7 @@ import { FileCard } from './file-card'
import { FolderCard } from './folder-card'
import { useFileActions } from './use-file-actions'
import { MobileFileBrowserContentSkeleton } from '../loading-skeletons'
import { useBatchFileStatus } from '@/ui/hooks/use-batch-file-status'
export interface MobileFileBrowserProps {
teamId: string
@@ -125,6 +126,7 @@ export function MobileFileBrowser({
allowDownload = true,
}: MobileFileBrowserProps) {
const queryClient = useQueryClient()
useBatchFileStatus({ teamId, items: files })
const { canEdit, canAdmin } = usePermissions(projectId)
const fileInputRef = useRef<HTMLInputElement>(null)
const folderInputRef = useRef<HTMLInputElement>(null)
@@ -200,6 +202,47 @@ export function MobileFileBrowser({
// Upload actions
const startTask = useUploadStore((state) => state.startTask)
const throttledInvalidateSearch = useMemo(() => {
let timer: ReturnType<typeof setTimeout> | null = null
let lastRun = 0
const THROTTLE_MS = 2500
const run = () => {
const now = Date.now()
if (now - lastRun >= THROTTLE_MS) {
lastRun = now
queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
} else if (!timer) {
timer = setTimeout(
() => {
timer = null
lastRun = Date.now()
queryClient.invalidateQueries({
queryKey: ['search', teamId, assetId],
})
},
THROTTLE_MS - (now - lastRun),
)
}
}
const cancel = () => {
if (timer) {
clearTimeout(timer)
timer = null
}
}
const reset = () => {
cancel()
lastRun = Date.now()
}
return Object.assign(run, { cancel, reset })
}, [queryClient, teamId, assetId])
const $createUploadTask = client.api.teams[':teamId'].upload.tasks.$post
const { mutate: createUploadTaskMutation } = useMutation<
InferResponseType<typeof $createUploadTask>,
@@ -218,7 +261,7 @@ export function MobileFileBrowser({
onError: () => {
toast.error(m.failed_upload_file())
},
onSuccess: async (data, _variables, context) => {
onSuccess: async (data, variables, context) => {
const currentFiles = context?.files || []
const filesProgressInfo = currentFiles
.map((f) => {
@@ -240,22 +283,32 @@ export function MobileFileBrowser({
startTask(data.taskId, taskName, filesProgressInfo)
}
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
const hasCreatedFolders = (variables.json.files as FileNode[]).some(
(f) => f.children && f.children.length > 0,
)
if (hasCreatedFolders) {
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
}
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] })
throttledInvalidateSearch.reset()
await uploadFilesWithUppy({
files: currentFiles,
taskId: data.taskId!,
teamId: teamId!,
storageBackend: (data.storageBackend as 's3' | 'local') || 'local',
createdAssets: data.createdAssets,
presignedUrls: data.presignedUrls,
onFileFinished: async () => {
await queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
},
})
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] })
try {
await uploadFilesWithUppy({
files: currentFiles,
taskId: data.taskId!,
teamId: teamId!,
storageBackend: (data.storageBackend as 's3' | 'local') || 'local',
createdAssets: data.createdAssets,
presignedUrls: data.presignedUrls,
onFileFinished: () => {
throttledInvalidateSearch()
},
})
} finally {
throttledInvalidateSearch.cancel()
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] })
}
},
})
@@ -0,0 +1,158 @@
// @vitest-environment happy-dom
import { renderHook, waitFor } from '@testing-library/react'
import React from 'react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import type { AssetInfo } from '@shumai/dtos'
import { useBatchFileStatus } from './use-batch-file-status'
import { client } from '@/ui/api/client'
import { useUploadStore } from '@/ui/stores/upload'
vi.mock('@/ui/api/client', () => ({
client: {
api: {
files: {
batch: {
$post: vi.fn(),
},
},
},
},
}))
describe('useBatchFileStatus', () => {
let queryClient: QueryClient
beforeEach(() => {
useUploadStore.setState({ fileProgress: {} })
queryClient = new QueryClient({
defaultOptions: {
queries: { retry: false },
},
})
vi.clearAllMocks()
})
afterEach(() => {
vi.restoreAllMocks()
})
const wrapper = ({ children }: { children: React.ReactNode }) =>
React.createElement(QueryClientProvider, { client: queryClient }, children)
it('does not poll when items array is empty or no files are processing', () => {
const items: AssetInfo[] = [
{ id: 'f1', type: 'file', status: 'processed' } as AssetInfo,
{ id: 'f2', type: 'file', status: 'error' } as AssetInfo,
]
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
expect(client.api.files.batch.$post).not.toHaveBeenCalled()
})
it('does NOT poll when files are in "uploading" status and upload is in progress', () => {
const items: AssetInfo[] = [
{ id: 'f1', type: 'file', status: 'uploading' } as AssetInfo,
{ id: 'f2', type: 'file', status: 'uploading' } as AssetInfo,
]
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
expect(client.api.files.batch.$post).not.toHaveBeenCalled()
})
it('polls batch API when file has status "uploading" but upload is completed in store', async () => {
useUploadStore.setState({
fileProgress: {
f1: { fileId: 'f1', name: 'f1.mp4', loaded: 1000, total: 1000, status: 'completed' },
},
})
const items: AssetInfo[] = [
{ id: 'f1', type: 'file', status: 'uploading' } as AssetInfo,
{ id: 'f2', type: 'file', status: 'uploading' } as AssetInfo,
]
const updatedAssets: AssetInfo[] = [
{ id: 'f1', type: 'file', status: 'uploaded', name: 'f1.mp4' } as AssetInfo,
]
const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries')
vi.mocked(client.api.files.batch.$post).mockResolvedValue({
ok: true,
json: async () => updatedAssets,
} as unknown as Awaited<ReturnType<typeof client.api.files.batch.$post>>)
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
await waitFor(() => {
expect(client.api.files.batch.$post).toHaveBeenCalledWith({
json: { ids: ['f1'] },
})
})
expect(queryClient.getQueryData(['file', 'team-1', 'f1'])).toEqual(updatedAssets[0])
// Should NOT invalidate search query for intermediate 'uploaded' state; card updates via setQueryData
expect(invalidateSpy).not.toHaveBeenCalledWith({ queryKey: ['search', 'team-1'] })
})
it('polls batch API when files are in "uploaded" or "processing" status and updates cache', async () => {
const items: AssetInfo[] = [
{ id: 'f1', type: 'file', status: 'uploaded' } as AssetInfo,
{ id: 'f2', type: 'file', status: 'processing' } as AssetInfo,
{ id: 'f3', type: 'file', status: 'uploading' } as AssetInfo,
]
const updatedAssets: AssetInfo[] = [
{ id: 'f1', type: 'file', status: 'processing', name: 'f1.mp4' } as AssetInfo,
{ id: 'f2', type: 'file', status: 'processed', name: 'f2.mp4' } as AssetInfo,
]
const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries')
vi.mocked(client.api.files.batch.$post).mockResolvedValue({
ok: true,
json: async () => updatedAssets,
} as unknown as Awaited<ReturnType<typeof client.api.files.batch.$post>>)
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
await waitFor(() => {
expect(client.api.files.batch.$post).toHaveBeenCalledWith({
json: { ids: ['f1', 'f2'] },
})
})
// Should update per-file cache in TanStack Query
expect(queryClient.getQueryData(['file', 'team-1', 'f1'])).toEqual(updatedAssets[0])
expect(queryClient.getQueryData(['file', 'team-1', 'f2'])).toEqual(updatedAssets[1])
// Should invalidate folder search query because f2 finished ('processed')
expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ['search', 'team-1'] })
})
it('chunks requests when processing items exceed 100', async () => {
const items: AssetInfo[] = Array.from({ length: 150 }, (_, i) => ({
id: `file-${i}`,
type: 'file',
status: 'processing',
})) as AssetInfo[]
vi.mocked(client.api.files.batch.$post).mockResolvedValue({
ok: true,
json: async () => [],
} as unknown as Awaited<ReturnType<typeof client.api.files.batch.$post>>)
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
await waitFor(() => {
expect(client.api.files.batch.$post).toHaveBeenCalledTimes(2)
})
const calls = vi.mocked(client.api.files.batch.$post).mock.calls
expect(calls[0][0].json.ids).toHaveLength(100)
expect(calls[1][0].json.ids).toHaveLength(50)
})
})
@@ -0,0 +1,96 @@
import { useQuery, useQueryClient } from '@tanstack/react-query'
import { useMemo, useRef } from 'react'
import type { AssetInfo } from '@shumai/dtos'
import { client } from '@/ui/api/client'
import { useUploadStore } from '@/ui/stores/upload'
export const BATCH_SIZE = 100
export const BATCH_POLL_INTERVAL_MS = 2000
export const BATCH_SEARCH_INVALIDATE_THROTTLE_MS = 5000
export interface UseBatchFileStatusOptions {
teamId?: string
items: AssetInfo[]
}
export function useBatchFileStatus({ teamId, items }: UseBatchFileStatusOptions): void {
const queryClient = useQueryClient()
const fileProgress = useUploadStore((state) => state.fileProgress)
const lastInvalidateTimeRef = useRef(0)
const processingIds = useMemo(() => {
if (!teamId) return []
return items
.filter((item) => {
if (!item.id || (item.type !== 'file' && item.type !== 'version_stack')) return false
const isUploadedOrProcessing = item.status === 'uploaded' || item.status === 'processing'
const progress = fileProgress[item.id]
const isUploadCompleted =
item.status === 'uploading' &&
(progress?.status === 'completed' ||
(progress?.total != null && progress.total > 0 && progress.loaded >= progress.total))
return isUploadedOrProcessing || isUploadCompleted
})
.map((item) => item.id!)
}, [items, teamId, fileProgress])
const processingIdsRef = useRef(processingIds)
processingIdsRef.current = processingIds
const queryKey = useMemo(() => {
return ['files', 'batch', teamId]
}, [teamId])
useQuery({
queryKey,
queryFn: async () => {
const currentIds = processingIdsRef.current
if (currentIds.length === 0) return []
const chunks: string[][] = []
for (let i = 0; i < currentIds.length; i += BATCH_SIZE) {
chunks.push(currentIds.slice(i, i + BATCH_SIZE))
}
const results = await Promise.all(
chunks.map(async (chunkIds) => {
const res = await client.api.files.batch.$post({
json: { ids: chunkIds },
})
if (!res.ok) throw new Error('Failed to fetch batch file statuses')
return (await res.json()) as unknown as AssetInfo[]
}),
)
const assets = results.flat()
for (const asset of assets) {
if (asset.id) {
queryClient.setQueryData(['file', teamId, asset.id], asset)
}
}
const anyFinished = assets.some(
(a) => a.status === 'processed' || a.status === 'failed' || a.status === 'error',
)
const allFinished =
assets.length > 0 &&
assets.every(
(a) => a.status === 'processed' || a.status === 'failed' || a.status === 'error',
)
const now = Date.now()
if (
allFinished ||
(anyFinished && now - lastInvalidateTimeRef.current >= BATCH_SEARCH_INVALIDATE_THROTTLE_MS)
) {
lastInvalidateTimeRef.current = now
queryClient.invalidateQueries({ queryKey: ['search', teamId] })
}
return assets
},
enabled: !!teamId && processingIds.length > 0,
refetchInterval: BATCH_POLL_INTERVAL_MS,
})
}