mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-19 14:16:29 +00:00
feat(file): optimize upload and transcoding status polling with batch api (#512)
* feat(file): optimize upload and transcoding status polling with batch api * fix(webui): throttle search invalidation during upload and fix stuck 100% progress ring * fix(webui): deduplicate upload search invalidation and stabilize batch polling
This commit is contained in:
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Changed
|
||||
|
||||
- **upload**: Optimize file status polling during upload and transcoding by eliminating redundant polling during active uploads and batching transcoding status checks across multiple files
|
||||
- **agent**: ⚠️ **Important Behavior Change**: Autofill and Embedding agents will no longer post completion comments when finishing successfully to prevent clutter in asset comment feeds. Comments are now only posted if a task encounters an error.
|
||||
|
||||
## [0.6.2] - 2026-10-29
|
||||
|
||||
@@ -9,6 +9,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { authMiddleware } from './middleware/auth'
|
||||
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import {
|
||||
renameAssetOp,
|
||||
resolveCommentOp,
|
||||
@@ -72,6 +73,7 @@ describe('file api', () => {
|
||||
vi.mocked(authzService.hasPermission).mockClear()
|
||||
|
||||
vi.spyOn(assetService, 'getAsset').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'listAssetsByIds').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'getAssetContext').mockResolvedValue({ teamId: 'test-team' })
|
||||
vi.spyOn(assetService, 'updateAssetName').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'deleteAssets').mockImplementation(vi.fn())
|
||||
@@ -126,6 +128,78 @@ describe('file api', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /files/batch', () => {
|
||||
it('returns empty array when ids is empty', async () => {
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: [] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual([])
|
||||
expect(assetService.listAssetsByIds).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns asset list for accessible IDs', async () => {
|
||||
const assets = [
|
||||
{ id: 'f1', name: 'file-1.mp4', status: 'processing' },
|
||||
{ id: 'f2', name: 'file-2.mp4', status: 'processed' },
|
||||
]
|
||||
vi.mocked(assetService.listAssetsByIds).mockResolvedValue(assets as unknown as AssetInfo[])
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['f1', 'f2'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual(assets)
|
||||
expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1', 'f2'])
|
||||
})
|
||||
|
||||
it('gracefully omits unauthorized or missing assets without failing the batch', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockImplementation(async (req) => {
|
||||
if (req.id === 'unauthorized-id') {
|
||||
throw new HTTPException(403, { message: 'Permission denied' })
|
||||
}
|
||||
})
|
||||
const accessibleAssets = [{ id: 'f1', name: 'file-1.mp4', status: 'processing' }]
|
||||
vi.mocked(assetService.listAssetsByIds).mockResolvedValue(
|
||||
accessibleAssets as unknown as AssetInfo[],
|
||||
)
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['f1', 'unauthorized-id'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual(accessibleAssets)
|
||||
expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1'])
|
||||
})
|
||||
|
||||
it('rejects with 400 when more than 100 IDs are requested', async () => {
|
||||
const tooManyIds = Array.from({ length: 101 }, (_, i) => `id-${i}`)
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: tooManyIds }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
})
|
||||
})
|
||||
|
||||
it('PUT /files/:fileId runs rename_asset', async () => {
|
||||
const summary = { id: 'test-id', name: 'new-name', type: 'file', size: 10, projectId: 'p1' }
|
||||
vi.mocked(runOperation).mockResolvedValue(summary)
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
import {
|
||||
createCommentRequestSchema,
|
||||
deleteFilesRequestSchema,
|
||||
batchGetFilesRequestSchema,
|
||||
paginationParamsSchema,
|
||||
restoreFilesRequestSchema,
|
||||
updateAssetMetadataRequestSchema,
|
||||
@@ -39,6 +40,40 @@ import { z } from 'zod'
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
|
||||
const route = new Hono<{ Variables: { user: User } }>()
|
||||
.post('/files/batch', zValidator('json', batchGetFilesRequestSchema), async (c) => {
|
||||
const user = c.get('user')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
if (req.ids.length === 0) {
|
||||
return c.json([])
|
||||
}
|
||||
|
||||
const accessibleIds: string[] = []
|
||||
await Promise.all(
|
||||
req.ids.map(async (id) => {
|
||||
try {
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id,
|
||||
})
|
||||
accessibleIds.push(id)
|
||||
} catch {
|
||||
// Permissive: silently ignore missing or unauthorized IDs
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
if (accessibleIds.length === 0) {
|
||||
return c.json([])
|
||||
}
|
||||
|
||||
const accessibleSet = new Set(accessibleIds)
|
||||
const orderedAccessibleIds = req.ids.filter((id) => accessibleSet.has(id))
|
||||
const assets = await assetService.listAssetsByIds(orderedAccessibleIds)
|
||||
return c.json(assets)
|
||||
})
|
||||
.get('/files/:fileId', async (c) => {
|
||||
const fileId = c.req.param('fileId')
|
||||
const user = c.get('user')
|
||||
|
||||
@@ -329,6 +329,11 @@ export const deleteFilesRequestSchema = z.object({
|
||||
})
|
||||
export type DeleteFilesRequest = z.infer<typeof deleteFilesRequestSchema>
|
||||
|
||||
export const batchGetFilesRequestSchema = z.object({
|
||||
ids: z.array(z.string()).max(100),
|
||||
})
|
||||
export type BatchGetFilesRequest = z.infer<typeof batchGetFilesRequestSchema>
|
||||
|
||||
export const restoreFilesRequestSchema = z.object({
|
||||
ids: z.array(z.string()),
|
||||
})
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
AssetInfo,
|
||||
CollectionInfo,
|
||||
CreateUploadTaskRequest,
|
||||
FileNode,
|
||||
SearchCondition,
|
||||
SearchSort,
|
||||
ShareLinkInfo,
|
||||
@@ -57,6 +58,7 @@ import { FileBrowserGridView } from './grid-view'
|
||||
import { FileBrowserListView } from './list-view'
|
||||
import { FileBrowserToolbar } from './toolbar'
|
||||
import { useFileActions } from './use-file-actions'
|
||||
import { useBatchFileStatus } from '@/ui/hooks/use-batch-file-status'
|
||||
|
||||
interface FileBrowserProps {
|
||||
teamId: string
|
||||
@@ -260,6 +262,8 @@ export function FileBrowser({
|
||||
return [...filteredLocal, ...files]
|
||||
}, [files, localUploadingFiles, fileProgress])
|
||||
|
||||
useBatchFileStatus({ teamId, items: displayedFiles })
|
||||
|
||||
const activePreviewItem = useMemo(() => {
|
||||
if (selectedIds.size === 0) return null
|
||||
if (internalSelectedItem && selectedIds.has(internalSelectedItem.id!)) {
|
||||
@@ -533,6 +537,47 @@ export function FileBrowser({
|
||||
},
|
||||
})
|
||||
|
||||
const throttledInvalidateSearch = useMemo(() => {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
let lastRun = 0
|
||||
const THROTTLE_MS = 2500
|
||||
|
||||
const run = () => {
|
||||
const now = Date.now()
|
||||
if (now - lastRun >= THROTTLE_MS) {
|
||||
lastRun = now
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
} else if (!timer) {
|
||||
timer = setTimeout(
|
||||
() => {
|
||||
timer = null
|
||||
lastRun = Date.now()
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
},
|
||||
THROTTLE_MS - (now - lastRun),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const cancel = () => {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
timer = null
|
||||
}
|
||||
}
|
||||
|
||||
const reset = () => {
|
||||
cancel()
|
||||
lastRun = Date.now()
|
||||
}
|
||||
|
||||
return Object.assign(run, { cancel, reset })
|
||||
}, [queryClient, teamId, assetId])
|
||||
|
||||
const $createUploadTask = client.api.teams[':teamId'].upload.tasks.$post
|
||||
const { mutate: createUploadTaskMutation } = useMutation<
|
||||
InferResponseType<typeof $createUploadTask>,
|
||||
@@ -623,13 +668,19 @@ export function FileBrowser({
|
||||
setLocalUploadingFiles((prev) => [...newLocalFiles, ...prev])
|
||||
}
|
||||
|
||||
// Invalidate queries immediately so any newly created folders appear
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
// Invalidate queries immediately only if newly created folders appear
|
||||
const hasCreatedFolders = (variables.json.files as FileNode[]).some(
|
||||
(f) => f.children && f.children.length > 0,
|
||||
)
|
||||
if (hasCreatedFolders) {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
}
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['teams', teamId, 'upload', 'tasks'],
|
||||
})
|
||||
throttledInvalidateSearch.reset()
|
||||
|
||||
try {
|
||||
await uploadFilesWithUppy({
|
||||
@@ -639,14 +690,13 @@ export function FileBrowser({
|
||||
storageBackend: (data.storageBackend as 's3' | 'local') || 'local',
|
||||
createdAssets: data.createdAssets,
|
||||
presignedUrls: data.presignedUrls,
|
||||
onFileFinished: async (fileId) => {
|
||||
await queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
onFileFinished: (fileId) => {
|
||||
setLocalUploadingFiles((prev) => prev.filter((f) => f.id !== fileId))
|
||||
throttledInvalidateSearch()
|
||||
},
|
||||
})
|
||||
} finally {
|
||||
throttledInvalidateSearch.cancel()
|
||||
if (batchIds) {
|
||||
setLocalUploadingFiles((prev) => prev.filter((f) => !batchIds!.has(f.id!)))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { FileCard } from './file-card'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { useUploadStore } from '@/ui/stores/upload'
|
||||
|
||||
vi.mock('@dnd-kit/react', () => ({
|
||||
useDraggable: () => ({
|
||||
@@ -341,6 +342,35 @@ describe('FileCard', () => {
|
||||
expect(screen.queryByText(/Alice/i)).toBeNull()
|
||||
})
|
||||
|
||||
it('transitions from uploading to preparing circle when upload store marks file completed', () => {
|
||||
useUploadStore.setState({
|
||||
fileProgress: {
|
||||
'file-uploading': {
|
||||
fileId: 'file-uploading',
|
||||
name: 'clip.mp4',
|
||||
loaded: 1000,
|
||||
total: 1000,
|
||||
status: 'completed',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const uploadingItem: AssetInfo = {
|
||||
...fileItem,
|
||||
id: 'file-uploading',
|
||||
status: 'uploading',
|
||||
creator: { id: 'u1', name: 'Alice' },
|
||||
} as AssetInfo
|
||||
|
||||
renderComponent({ item: uploadingItem })
|
||||
|
||||
expect(screen.queryByTestId('file-card-preview-media')).toBeNull()
|
||||
const circle = screen.getByTestId('file-card-preparing-circle')
|
||||
expect(circle.tagName.toLowerCase()).toBe('svg')
|
||||
expect(screen.getByText(/Preparing|准备中/i)).toBeTruthy()
|
||||
expect(screen.queryByText(/Alice/i)).toBeNull()
|
||||
})
|
||||
|
||||
it('shows a preparing circle and "Preparing..." when processing without a preview yet', () => {
|
||||
const processingItem: AssetInfo = {
|
||||
...fileItem,
|
||||
|
||||
@@ -119,11 +119,6 @@ export function FileCard({
|
||||
: 0
|
||||
: 0
|
||||
|
||||
const shouldPoll =
|
||||
(item.type === 'file' || item.type === 'version_stack') &&
|
||||
!!teamId &&
|
||||
(item.status === 'uploading' || item.status === 'uploaded' || item.status === 'processing')
|
||||
|
||||
const { data: polledItem } = useQuery({
|
||||
queryKey: ['file', teamId, item.id],
|
||||
queryFn: async () => {
|
||||
@@ -133,13 +128,7 @@ export function FileCard({
|
||||
if (!res.ok) throw new Error('failed to fetch file')
|
||||
return (await res.json()) as unknown as AssetInfo
|
||||
},
|
||||
enabled: shouldPoll,
|
||||
refetchInterval: (query: unknown) => {
|
||||
const data = (query as { state: { data: { status?: string } } }).state.data
|
||||
return data?.status === 'processed' || data?.status === 'failed' || data?.status === 'error'
|
||||
? false
|
||||
: 1000
|
||||
},
|
||||
enabled: false,
|
||||
})
|
||||
|
||||
const displayItem = useMemo(() => {
|
||||
@@ -277,16 +266,16 @@ export function FileCard({
|
||||
// since the poster/sprite are generated and persisted ahead of the proxy transcodes.
|
||||
const hasPreview = Boolean(displayItem.preview?.thumbnailUrl || displayItem.preview?.spriteUrl)
|
||||
const isFailed = displayItem.status === 'failed'
|
||||
const isProcessing = displayItem.status === 'processing' || displayItem.status === 'uploaded'
|
||||
const isUploadDone = fileUploadState?.status === 'completed' || uploadPercent >= 100
|
||||
const isUploading = displayItem.status === 'uploading' && !isUploadDone
|
||||
const isProcessing =
|
||||
displayItem.status === 'processing' ||
|
||||
displayItem.status === 'uploaded' ||
|
||||
(displayItem.status === 'uploading' && isUploadDone)
|
||||
|
||||
// While uploading/transcoding, the creator row is replaced by a short status label so the real
|
||||
// date/author only appears once the asset is ready or failed.
|
||||
const statusText =
|
||||
displayItem.status === 'uploading'
|
||||
? m.uploading()
|
||||
: displayItem.status === 'uploaded' || displayItem.status === 'processing'
|
||||
? m.preparing()
|
||||
: null
|
||||
const statusText = isUploading ? m.uploading() : isProcessing ? m.preparing() : null
|
||||
|
||||
const failedErrorMessage = displayItem.media?.error || m.processing_failed()
|
||||
|
||||
@@ -393,7 +382,7 @@ export function FileCard({
|
||||
</div>
|
||||
|
||||
<div className="relative aspect-square overflow-hidden bg-muted/30">
|
||||
{displayItem.status === 'uploading' ? (
|
||||
{isUploading ? (
|
||||
<div className="flex h-full w-full items-center justify-center bg-background/50">
|
||||
<ProgressCircle progress={uploadPercent} className="w-16 h-16 z-10" />
|
||||
</div>
|
||||
|
||||
@@ -74,11 +74,6 @@ export function FileListItem({
|
||||
: 0
|
||||
: 0
|
||||
|
||||
const shouldPoll =
|
||||
(item.type === 'file' || item.type === 'version_stack') &&
|
||||
!!teamId &&
|
||||
(item.status === 'uploading' || item.status === 'uploaded' || item.status === 'processing')
|
||||
|
||||
const { data: polledItem } = useQuery({
|
||||
queryKey: ['file', teamId, item.id],
|
||||
queryFn: async () => {
|
||||
@@ -88,13 +83,7 @@ export function FileListItem({
|
||||
if (!res.ok) throw new Error('failed to fetch file')
|
||||
return (await res.json()) as unknown as AssetInfo
|
||||
},
|
||||
enabled: shouldPoll,
|
||||
refetchInterval: (query: unknown) => {
|
||||
const data = (query as { state: { data: { status?: string } } }).state.data
|
||||
return data?.status === 'processed' || data?.status === 'failed' || data?.status === 'error'
|
||||
? false
|
||||
: 1000
|
||||
},
|
||||
enabled: false,
|
||||
})
|
||||
|
||||
const displayItem = useMemo(() => {
|
||||
@@ -111,6 +100,9 @@ export function FileListItem({
|
||||
return polledItem || item
|
||||
}, [item, polledItem])
|
||||
|
||||
const isUploadDone = fileUploadState?.status === 'completed' || uploadPercent >= 100
|
||||
const isUploading = displayItem.status === 'uploading' && !isUploadDone
|
||||
|
||||
// dnd-kit hooks
|
||||
const { ref: setDraggableRef, isDragging: isDraggableDragging } = useDraggable({
|
||||
id: `browser:${displayItem.id!}`,
|
||||
@@ -220,7 +212,7 @@ export function FileListItem({
|
||||
className="h-4 w-4 shrink-0 data-[state=checked]:bg-primary data-[state=checked]:border-primary"
|
||||
/>
|
||||
<div className="w-8 h-8 shrink-0 flex items-center justify-center bg-muted rounded overflow-hidden">
|
||||
{displayItem.status === 'uploading' ? (
|
||||
{isUploading ? (
|
||||
<ProgressCircle progress={uploadPercent} className="w-7 h-7" />
|
||||
) : displayItem.status === 'failed' && !displayItem.preview?.thumbnailUrl ? (
|
||||
<AlertCircle className="h-4 w-4 text-destructive" />
|
||||
|
||||
@@ -5,7 +5,7 @@ import { usePermissions } from '@/ui/hooks/use-permissions'
|
||||
import { m } from '@/ui/paraglide/messages.js'
|
||||
import { useUploadStore } from '@/ui/stores/upload'
|
||||
import { uploadFilesWithUppy } from '@/ui/lib/uploader'
|
||||
import type { AssetInfo, CreateUploadTaskRequest } from '@shumai/dtos'
|
||||
import type { AssetInfo, CreateUploadTaskRequest, FileNode } from '@shumai/dtos'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import type { InferRequestType, InferResponseType } from 'hono/client'
|
||||
import { formatSize } from '@/ui/lib/format'
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
Plus,
|
||||
Upload,
|
||||
} from 'lucide-react'
|
||||
import React, { useRef, useState } from 'react'
|
||||
import React, { useMemo, useRef, useState } from 'react'
|
||||
import { useInView } from 'react-intersection-observer'
|
||||
import { toast } from 'sonner'
|
||||
import { ulid } from 'ulid'
|
||||
@@ -54,6 +54,7 @@ import { FileCard } from './file-card'
|
||||
import { FolderCard } from './folder-card'
|
||||
import { useFileActions } from './use-file-actions'
|
||||
import { MobileFileBrowserContentSkeleton } from '../loading-skeletons'
|
||||
import { useBatchFileStatus } from '@/ui/hooks/use-batch-file-status'
|
||||
|
||||
export interface MobileFileBrowserProps {
|
||||
teamId: string
|
||||
@@ -125,6 +126,7 @@ export function MobileFileBrowser({
|
||||
allowDownload = true,
|
||||
}: MobileFileBrowserProps) {
|
||||
const queryClient = useQueryClient()
|
||||
useBatchFileStatus({ teamId, items: files })
|
||||
const { canEdit, canAdmin } = usePermissions(projectId)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const folderInputRef = useRef<HTMLInputElement>(null)
|
||||
@@ -200,6 +202,47 @@ export function MobileFileBrowser({
|
||||
// Upload actions
|
||||
const startTask = useUploadStore((state) => state.startTask)
|
||||
|
||||
const throttledInvalidateSearch = useMemo(() => {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
let lastRun = 0
|
||||
const THROTTLE_MS = 2500
|
||||
|
||||
const run = () => {
|
||||
const now = Date.now()
|
||||
if (now - lastRun >= THROTTLE_MS) {
|
||||
lastRun = now
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
} else if (!timer) {
|
||||
timer = setTimeout(
|
||||
() => {
|
||||
timer = null
|
||||
lastRun = Date.now()
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ['search', teamId, assetId],
|
||||
})
|
||||
},
|
||||
THROTTLE_MS - (now - lastRun),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const cancel = () => {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
timer = null
|
||||
}
|
||||
}
|
||||
|
||||
const reset = () => {
|
||||
cancel()
|
||||
lastRun = Date.now()
|
||||
}
|
||||
|
||||
return Object.assign(run, { cancel, reset })
|
||||
}, [queryClient, teamId, assetId])
|
||||
|
||||
const $createUploadTask = client.api.teams[':teamId'].upload.tasks.$post
|
||||
const { mutate: createUploadTaskMutation } = useMutation<
|
||||
InferResponseType<typeof $createUploadTask>,
|
||||
@@ -218,7 +261,7 @@ export function MobileFileBrowser({
|
||||
onError: () => {
|
||||
toast.error(m.failed_upload_file())
|
||||
},
|
||||
onSuccess: async (data, _variables, context) => {
|
||||
onSuccess: async (data, variables, context) => {
|
||||
const currentFiles = context?.files || []
|
||||
const filesProgressInfo = currentFiles
|
||||
.map((f) => {
|
||||
@@ -240,22 +283,32 @@ export function MobileFileBrowser({
|
||||
startTask(data.taskId, taskName, filesProgressInfo)
|
||||
}
|
||||
|
||||
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
|
||||
const hasCreatedFolders = (variables.json.files as FileNode[]).some(
|
||||
(f) => f.children && f.children.length > 0,
|
||||
)
|
||||
if (hasCreatedFolders) {
|
||||
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
|
||||
}
|
||||
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] })
|
||||
throttledInvalidateSearch.reset()
|
||||
|
||||
await uploadFilesWithUppy({
|
||||
files: currentFiles,
|
||||
taskId: data.taskId!,
|
||||
teamId: teamId!,
|
||||
storageBackend: (data.storageBackend as 's3' | 'local') || 'local',
|
||||
createdAssets: data.createdAssets,
|
||||
presignedUrls: data.presignedUrls,
|
||||
onFileFinished: async () => {
|
||||
await queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
|
||||
},
|
||||
})
|
||||
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
|
||||
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] })
|
||||
try {
|
||||
await uploadFilesWithUppy({
|
||||
files: currentFiles,
|
||||
taskId: data.taskId!,
|
||||
teamId: teamId!,
|
||||
storageBackend: (data.storageBackend as 's3' | 'local') || 'local',
|
||||
createdAssets: data.createdAssets,
|
||||
presignedUrls: data.presignedUrls,
|
||||
onFileFinished: () => {
|
||||
throttledInvalidateSearch()
|
||||
},
|
||||
})
|
||||
} finally {
|
||||
throttledInvalidateSearch.cancel()
|
||||
queryClient.invalidateQueries({ queryKey: ['search', teamId, assetId] })
|
||||
queryClient.invalidateQueries({ queryKey: ['teams', teamId, 'upload', 'tasks'] })
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { renderHook, waitFor } from '@testing-library/react'
|
||||
import React from 'react'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import { useBatchFileStatus } from './use-batch-file-status'
|
||||
import { client } from '@/ui/api/client'
|
||||
import { useUploadStore } from '@/ui/stores/upload'
|
||||
|
||||
vi.mock('@/ui/api/client', () => ({
|
||||
client: {
|
||||
api: {
|
||||
files: {
|
||||
batch: {
|
||||
$post: vi.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
describe('useBatchFileStatus', () => {
|
||||
let queryClient: QueryClient
|
||||
|
||||
beforeEach(() => {
|
||||
useUploadStore.setState({ fileProgress: {} })
|
||||
queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: { retry: false },
|
||||
},
|
||||
})
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
const wrapper = ({ children }: { children: React.ReactNode }) =>
|
||||
React.createElement(QueryClientProvider, { client: queryClient }, children)
|
||||
|
||||
it('does not poll when items array is empty or no files are processing', () => {
|
||||
const items: AssetInfo[] = [
|
||||
{ id: 'f1', type: 'file', status: 'processed' } as AssetInfo,
|
||||
{ id: 'f2', type: 'file', status: 'error' } as AssetInfo,
|
||||
]
|
||||
|
||||
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
|
||||
|
||||
expect(client.api.files.batch.$post).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does NOT poll when files are in "uploading" status and upload is in progress', () => {
|
||||
const items: AssetInfo[] = [
|
||||
{ id: 'f1', type: 'file', status: 'uploading' } as AssetInfo,
|
||||
{ id: 'f2', type: 'file', status: 'uploading' } as AssetInfo,
|
||||
]
|
||||
|
||||
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
|
||||
|
||||
expect(client.api.files.batch.$post).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('polls batch API when file has status "uploading" but upload is completed in store', async () => {
|
||||
useUploadStore.setState({
|
||||
fileProgress: {
|
||||
f1: { fileId: 'f1', name: 'f1.mp4', loaded: 1000, total: 1000, status: 'completed' },
|
||||
},
|
||||
})
|
||||
|
||||
const items: AssetInfo[] = [
|
||||
{ id: 'f1', type: 'file', status: 'uploading' } as AssetInfo,
|
||||
{ id: 'f2', type: 'file', status: 'uploading' } as AssetInfo,
|
||||
]
|
||||
|
||||
const updatedAssets: AssetInfo[] = [
|
||||
{ id: 'f1', type: 'file', status: 'uploaded', name: 'f1.mp4' } as AssetInfo,
|
||||
]
|
||||
|
||||
const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries')
|
||||
|
||||
vi.mocked(client.api.files.batch.$post).mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => updatedAssets,
|
||||
} as unknown as Awaited<ReturnType<typeof client.api.files.batch.$post>>)
|
||||
|
||||
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
|
||||
|
||||
await waitFor(() => {
|
||||
expect(client.api.files.batch.$post).toHaveBeenCalledWith({
|
||||
json: { ids: ['f1'] },
|
||||
})
|
||||
})
|
||||
|
||||
expect(queryClient.getQueryData(['file', 'team-1', 'f1'])).toEqual(updatedAssets[0])
|
||||
// Should NOT invalidate search query for intermediate 'uploaded' state; card updates via setQueryData
|
||||
expect(invalidateSpy).not.toHaveBeenCalledWith({ queryKey: ['search', 'team-1'] })
|
||||
})
|
||||
|
||||
it('polls batch API when files are in "uploaded" or "processing" status and updates cache', async () => {
|
||||
const items: AssetInfo[] = [
|
||||
{ id: 'f1', type: 'file', status: 'uploaded' } as AssetInfo,
|
||||
{ id: 'f2', type: 'file', status: 'processing' } as AssetInfo,
|
||||
{ id: 'f3', type: 'file', status: 'uploading' } as AssetInfo,
|
||||
]
|
||||
|
||||
const updatedAssets: AssetInfo[] = [
|
||||
{ id: 'f1', type: 'file', status: 'processing', name: 'f1.mp4' } as AssetInfo,
|
||||
{ id: 'f2', type: 'file', status: 'processed', name: 'f2.mp4' } as AssetInfo,
|
||||
]
|
||||
|
||||
const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries')
|
||||
|
||||
vi.mocked(client.api.files.batch.$post).mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => updatedAssets,
|
||||
} as unknown as Awaited<ReturnType<typeof client.api.files.batch.$post>>)
|
||||
|
||||
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
|
||||
|
||||
await waitFor(() => {
|
||||
expect(client.api.files.batch.$post).toHaveBeenCalledWith({
|
||||
json: { ids: ['f1', 'f2'] },
|
||||
})
|
||||
})
|
||||
|
||||
// Should update per-file cache in TanStack Query
|
||||
expect(queryClient.getQueryData(['file', 'team-1', 'f1'])).toEqual(updatedAssets[0])
|
||||
expect(queryClient.getQueryData(['file', 'team-1', 'f2'])).toEqual(updatedAssets[1])
|
||||
|
||||
// Should invalidate folder search query because f2 finished ('processed')
|
||||
expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ['search', 'team-1'] })
|
||||
})
|
||||
|
||||
it('chunks requests when processing items exceed 100', async () => {
|
||||
const items: AssetInfo[] = Array.from({ length: 150 }, (_, i) => ({
|
||||
id: `file-${i}`,
|
||||
type: 'file',
|
||||
status: 'processing',
|
||||
})) as AssetInfo[]
|
||||
|
||||
vi.mocked(client.api.files.batch.$post).mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => [],
|
||||
} as unknown as Awaited<ReturnType<typeof client.api.files.batch.$post>>)
|
||||
|
||||
renderHook(() => useBatchFileStatus({ teamId: 'team-1', items }), { wrapper })
|
||||
|
||||
await waitFor(() => {
|
||||
expect(client.api.files.batch.$post).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
const calls = vi.mocked(client.api.files.batch.$post).mock.calls
|
||||
expect(calls[0][0].json.ids).toHaveLength(100)
|
||||
expect(calls[1][0].json.ids).toHaveLength(50)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,96 @@
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useMemo, useRef } from 'react'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import { client } from '@/ui/api/client'
|
||||
import { useUploadStore } from '@/ui/stores/upload'
|
||||
|
||||
export const BATCH_SIZE = 100
|
||||
export const BATCH_POLL_INTERVAL_MS = 2000
|
||||
export const BATCH_SEARCH_INVALIDATE_THROTTLE_MS = 5000
|
||||
|
||||
export interface UseBatchFileStatusOptions {
|
||||
teamId?: string
|
||||
items: AssetInfo[]
|
||||
}
|
||||
|
||||
export function useBatchFileStatus({ teamId, items }: UseBatchFileStatusOptions): void {
|
||||
const queryClient = useQueryClient()
|
||||
const fileProgress = useUploadStore((state) => state.fileProgress)
|
||||
const lastInvalidateTimeRef = useRef(0)
|
||||
|
||||
const processingIds = useMemo(() => {
|
||||
if (!teamId) return []
|
||||
return items
|
||||
.filter((item) => {
|
||||
if (!item.id || (item.type !== 'file' && item.type !== 'version_stack')) return false
|
||||
const isUploadedOrProcessing = item.status === 'uploaded' || item.status === 'processing'
|
||||
const progress = fileProgress[item.id]
|
||||
const isUploadCompleted =
|
||||
item.status === 'uploading' &&
|
||||
(progress?.status === 'completed' ||
|
||||
(progress?.total != null && progress.total > 0 && progress.loaded >= progress.total))
|
||||
return isUploadedOrProcessing || isUploadCompleted
|
||||
})
|
||||
.map((item) => item.id!)
|
||||
}, [items, teamId, fileProgress])
|
||||
|
||||
const processingIdsRef = useRef(processingIds)
|
||||
processingIdsRef.current = processingIds
|
||||
|
||||
const queryKey = useMemo(() => {
|
||||
return ['files', 'batch', teamId]
|
||||
}, [teamId])
|
||||
|
||||
useQuery({
|
||||
queryKey,
|
||||
queryFn: async () => {
|
||||
const currentIds = processingIdsRef.current
|
||||
if (currentIds.length === 0) return []
|
||||
|
||||
const chunks: string[][] = []
|
||||
for (let i = 0; i < currentIds.length; i += BATCH_SIZE) {
|
||||
chunks.push(currentIds.slice(i, i + BATCH_SIZE))
|
||||
}
|
||||
|
||||
const results = await Promise.all(
|
||||
chunks.map(async (chunkIds) => {
|
||||
const res = await client.api.files.batch.$post({
|
||||
json: { ids: chunkIds },
|
||||
})
|
||||
if (!res.ok) throw new Error('Failed to fetch batch file statuses')
|
||||
return (await res.json()) as unknown as AssetInfo[]
|
||||
}),
|
||||
)
|
||||
|
||||
const assets = results.flat()
|
||||
|
||||
for (const asset of assets) {
|
||||
if (asset.id) {
|
||||
queryClient.setQueryData(['file', teamId, asset.id], asset)
|
||||
}
|
||||
}
|
||||
|
||||
const anyFinished = assets.some(
|
||||
(a) => a.status === 'processed' || a.status === 'failed' || a.status === 'error',
|
||||
)
|
||||
const allFinished =
|
||||
assets.length > 0 &&
|
||||
assets.every(
|
||||
(a) => a.status === 'processed' || a.status === 'failed' || a.status === 'error',
|
||||
)
|
||||
|
||||
const now = Date.now()
|
||||
if (
|
||||
allFinished ||
|
||||
(anyFinished && now - lastInvalidateTimeRef.current >= BATCH_SEARCH_INVALIDATE_THROTTLE_MS)
|
||||
) {
|
||||
lastInvalidateTimeRef.current = now
|
||||
queryClient.invalidateQueries({ queryKey: ['search', teamId] })
|
||||
}
|
||||
|
||||
return assets
|
||||
},
|
||||
enabled: !!teamId && processingIds.length > 0,
|
||||
refetchInterval: BATCH_POLL_INTERVAL_MS,
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user