From 77bd1422d8f5a8fe9f323d355163d637a90f5a79 Mon Sep 17 00:00:00 2001 From: Yiling-J Date: Sat, 31 Oct 2026 10:37:35 +0800 Subject: [PATCH] feat(upload): state-based retry tracking and resilient gateway error handling (#509) * feat(upload): state-based retry tracking and resilient gateway error handling * fix(upload): check non-retryable statuses before offline state in isNetworkIssue --- CHANGELOG.md | 2 +- .../file-browser/mobile-file-browser.test.tsx | 16 +- packages/webui/e2e/harness/main.tsx | 93 +++++- .../e2e/tests/uploader-resilience.spec.ts | 193 +++++++++++++ packages/webui/lib/uploader.test.ts | 266 ++++++++++++++++++ packages/webui/lib/uploader.ts | 224 +++++++++++---- 6 files changed, 727 insertions(+), 67 deletions(-) create mode 100644 packages/webui/e2e/tests/uploader-resilience.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 99c0379f..f2c3694a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added -- **upload**: Improve upload resilience during network drops and device sleep by adding continuous exponential backoff retries and extending upload link expiration to 12 hours +- **upload**: Improve upload resilience during network drops, device sleep, and transient gateway errors by adding state-based exponential backoff retries and extending upload link expiration to 12 hours ### Fixed diff --git a/packages/webui/components/file-browser/mobile-file-browser.test.tsx b/packages/webui/components/file-browser/mobile-file-browser.test.tsx index 3d55dd2d..42a1ff96 100644 --- a/packages/webui/components/file-browser/mobile-file-browser.test.tsx +++ b/packages/webui/components/file-browser/mobile-file-browser.test.tsx @@ -358,7 +358,7 @@ describe('MobileFileBrowser', () => { window.XMLHttpRequest = originalXhr }) - it('fails upload and confirms with error on network failure', async () => { + it('keeps upload active and does not fail task on network failure', async () => { const originalXhr = window.XMLHttpRequest const patchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({}) }) @@ -409,17 +409,13 @@ describe('MobileFileBrowser', () => { fireEvent.change(fileInput, { target: { files: [testFile] } }) await vi.waitFor(() => { - expect(patchMock).toHaveBeenCalledWith( - expect.objectContaining({ - param: { teamId: 'team-1', taskId: 'task-3' }, - json: expect.objectContaining({ - fileId: 'server-file-3', - errorMessage: expect.any(String), - }), - }), - ) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((client.api.teams[':teamId'].upload.tasks as any)['$post']).toHaveBeenCalled() }) + // Network failures are retried automatically with backoff and do not fail task on server + expect(patchMock).not.toHaveBeenCalled() + window.XMLHttpRequest = originalXhr }) diff --git a/packages/webui/e2e/harness/main.tsx b/packages/webui/e2e/harness/main.tsx index a0fc2e74..12d113d8 100644 --- a/packages/webui/e2e/harness/main.tsx +++ b/packages/webui/e2e/harness/main.tsx @@ -1,10 +1,13 @@ -import { type ReactElement, useCallback, useRef, useState } from 'react' +import { type ReactElement, useCallback, useEffect, useRef, useState } from 'react' import ReactDOM from 'react-dom/client' import type { MediaController } from '@/ui/components/viewers/types' import VideoPlayer from '@/ui/components/viewers/video/video-viewer' import ModelViewer from '@/ui/components/viewers/model/model-viewer' import { calculateFrameCenterTime } from '@/ui/components/viewers/video/utils' import { useUiStore } from '@/ui/stores/ui' +import { uploadFilesWithUppy } from '@/ui/lib/uploader' +import { useUploadStore } from '@/ui/stores/upload' +import { Toaster } from 'sonner' import { SAMPLE_FRAME_RATE, sampleVideoAsset, @@ -185,9 +188,95 @@ function Harness(): ReactElement { ) } +function UploaderHarness(): ReactElement { + const [status, setStatus] = useState<'idle' | 'uploading' | 'completed' | 'failed'>('idle') + const [errorMessage, setErrorMessage] = useState(null) + const taskState = useUploadStore((state) => state.tasks['e2e-task-1']) + + useEffect(() => { + ;(window as unknown as Record)['__SHUMAI_UPLOAD_BASE_DELAY__'] = 100 + }, []) + + const startUpload = async (file: File) => { + setStatus('uploading') + setErrorMessage(null) + + const taskId = 'e2e-task-1' + const teamId = 'e2e-team-1' + const fileId = 'e2e-file-1' + + useUploadStore + .getState() + .startTask(taskId, 'e2e-task', [{ fileId, name: file.name, size: file.size }]) + + try { + await uploadFilesWithUppy({ + files: [{ id: 'temp-1', file }], + taskId, + teamId, + storageBackend: 's3', + createdAssets: [{ tempId: 'temp-1', assetId: fileId, key: `files/${file.name}` }], + }) + setStatus('completed') + } catch (err) { + setStatus('failed') + setErrorMessage(err instanceof Error ? err.message : String(err)) + } + } + + const handleFileChange = (e: React.ChangeEvent) => { + const file = e.target.files?.[0] + if (file) { + void startUpload(file) + } + } + + const handleSimulatedStart = () => { + const mockFile = new File(['mock content for test'], 'test-file.mp4', { + type: 'video/mp4', + }) + void startUpload(mockFile) + } + + const derivedStatus = taskState?.files['e2e-file-1']?.status ?? status + + return ( +
+ +

Uploader Resilience E2E Harness

+
+ + +
+ +
+
+ Status: {derivedStatus} +
+
+ Progress:{' '} + {taskState?.files['e2e-file-1']?.loaded ?? 0} +
+ {errorMessage &&
{errorMessage}
} +
+
+ ) +} + const rootElement = document.getElementById('root') if (!rootElement) { throw new Error('Harness root element not found') } -ReactDOM.createRoot(rootElement).render() +const currentView = + typeof window !== 'undefined' ? new URLSearchParams(window.location.search).get('view') : null + +ReactDOM.createRoot(rootElement).render( + currentView === 'uploader' ? : , +) diff --git a/packages/webui/e2e/tests/uploader-resilience.spec.ts b/packages/webui/e2e/tests/uploader-resilience.spec.ts new file mode 100644 index 00000000..78ae8312 --- /dev/null +++ b/packages/webui/e2e/tests/uploader-resilience.spec.ts @@ -0,0 +1,193 @@ +import { expect, test } from '@playwright/test' + +test.describe('Uploader Resilience E2E Tests', () => { + test.beforeEach(async ({ page }) => { + // Default route mocks for the isolated harness + await page.route('**/api/teams/**/upload/tasks/**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ ok: true }), + }), + ) + }) + + test('keeps upload active on network drop and automatically resumes on reconnection', async ({ + page, + }) => { + let releaseSign: () => void = () => {} + const signGate = new Promise((resolve) => { + releaseSign = resolve + }) + + let signCalls = 0 + await page.route('**/api/teams/**/upload/sign', async (route) => { + signCalls++ + if (signCalls === 1) { + // Wait until offline event is dispatched before releasing first sign call + await signGate + } + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }), + }) + }) + + await page.route('**/mock-upload', (route) => + route.fulfill({ + status: 200, + headers: { ['ETag']: '"test-etag"' }, + }), + ) + + await page.goto('/?view=uploader') + await expect(page.getByTestId('upload-status')).toHaveText('idle') + + // Start upload + await page.getByTestId('start-upload-btn').click() + await expect(page.getByTestId('upload-status')).toHaveText('uploading') + + // Simulate cutting the connection + await page.context().setOffline(true) + releaseSign() + + // Status remains 'uploading' during offline state and no error toast appears + await page.waitForTimeout(500) + await expect(page.getByTestId('upload-status')).toHaveText('uploading') + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0) + + // Reconnect network + await page.context().setOffline(false) + + // Upload automatically resumes and completes + await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 }) + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0) + }) + + test('retries with backoff on in-flight socket drop and completes when network recovers', async ({ + page, + }) => { + let signCalls = 0 + await page.route('**/api/teams/**/upload/sign', (route) => { + signCalls++ + if (signCalls <= 2) { + // In-flight connection reset / socket drop + route.abort('connectionreset') + } else { + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }), + }) + } + }) + + await page.route('**/mock-upload', (route) => + route.fulfill({ + status: 200, + headers: { ['ETag']: '"test-etag"' }, + }), + ) + + await page.goto('/?view=uploader') + await page.getByTestId('start-upload-btn').click() + + await expect(page.getByTestId('upload-status')).toHaveText('uploading') + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0) + + // After backoff retries, succeeds on attempt 3 + await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 }) + expect(signCalls).toBeGreaterThanOrEqual(3) + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0) + }) + + test('retries continuously on 502 Bad Gateway and recovers once server returns 200', async ({ + page, + }) => { + let signAttempts = 0 + await page.route('**/api/teams/**/upload/sign', (route) => { + signAttempts++ + if (signAttempts <= 2) { + // Return 502 Bad Gateway for the first 2 attempts + route.fulfill({ + status: 502, + contentType: 'text/html', + body: '502 Bad Gateway', + }) + } else { + // Server recovers on attempt 3 + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }), + }) + } + }) + + await page.route('**/mock-upload', (route) => + route.fulfill({ + status: 200, + headers: { ['ETag']: '"test-etag"' }, + }), + ) + + await page.goto('/?view=uploader') + await page.getByTestId('start-upload-btn').click() + + // Verify it stays uploading without failing on 502 + await expect(page.getByTestId('upload-status')).toHaveText('uploading') + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0) + + // Once server recovers, upload completes + await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 }) + expect(signAttempts).toBeGreaterThanOrEqual(3) + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0) + }) + + test('halts immediately without retrying on 500 Internal Server Error', async ({ page }) => { + let signAttempts = 0 + await page.route('**/api/teams/**/upload/sign', (route) => { + signAttempts++ + route.fulfill({ + status: 500, + contentType: 'application/json', + body: JSON.stringify({ message: 'Internal Server Error' }), + }) + }) + + await page.goto('/?view=uploader') + await page.getByTestId('start-upload-btn').click() + + // Must fail immediately with error toast + await expect(page.getByTestId('upload-status')).toHaveText('failed', { timeout: 5_000 }) + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toBeVisible({ + timeout: 5_000, + }) + // 0 retries (attempt count must be 1) + expect(signAttempts).toBe(1) + }) + + test('halts immediately without retrying on 403 Forbidden', async ({ page }) => { + let signAttempts = 0 + await page.route('**/api/teams/**/upload/sign', (route) => { + signAttempts++ + route.fulfill({ + status: 403, + contentType: 'application/json', + body: JSON.stringify({ message: 'Forbidden' }), + }) + }) + + await page.goto('/?view=uploader') + await page.getByTestId('start-upload-btn').click() + + // Must fail immediately with error toast + await expect(page.getByTestId('upload-status')).toHaveText('failed', { timeout: 5_000 }) + await expect(page.locator('[data-sonner-toast][data-type="error"]')).toBeVisible({ + timeout: 5_000, + }) + // 0 retries (attempt count must be 1) + expect(signAttempts).toBe(1) + }) +}) diff --git a/packages/webui/lib/uploader.test.ts b/packages/webui/lib/uploader.test.ts index acf1cdfe..8e59b213 100644 --- a/packages/webui/lib/uploader.test.ts +++ b/packages/webui/lib/uploader.test.ts @@ -8,6 +8,8 @@ import { calculateBackoffDelay, MAX_RETRY_DELAY_MS, BASE_RETRY_DELAY_MS, + UploadNetworkError, + UploadHttpError, } from './uploader' import { acquireWakeLock, releaseWakeLock } from './wake-lock' import { useUploadStore } from '@/ui/stores/upload' @@ -206,9 +208,59 @@ describe('Uploader', () => { Object.defineProperty(navigator, 'onLine', { value: false, configurable: true }) expect(isNetworkIssue(new Error('Any error'))).toBe(true) expect(isNetworkIssue()).toBe(true) + // Non-retryable statuses must return false even while offline + expect( + isNetworkIssue( + new UploadHttpError('Internal Server Error', { status: 500, isTransient: false }), + ), + ).toBe(false) + expect( + isNetworkIssue(new UploadHttpError('Forbidden', { status: 403, isTransient: false })), + ).toBe(false) + expect(isNetworkIssue({ status: 500 })).toBe(false) + expect(isNetworkIssue({ status: 403 })).toBe(false) + expect(isNetworkIssue({ request: { status: 500 } })).toBe(false) + expect(isNetworkIssue({ request: { status: 403 } })).toBe(false) + expect(isNetworkIssue('upload failed with status: 500')).toBe(false) + expect(isNetworkIssue('upload failed with status: 403')).toBe(false) + // Transient statuses while offline return true + expect( + isNetworkIssue(new UploadHttpError('Bad Gateway', { status: 502, isTransient: true })), + ).toBe(true) + expect(isNetworkIssue({ status: 502 })).toBe(true) Object.defineProperty(navigator, 'onLine', { value: true, configurable: true }) expect(isNetworkIssue(new Error('Failed to fetch'))).toBe(true) + expect(isNetworkIssue(new TypeError('Failed to fetch'))).toBe(true) + expect(isNetworkIssue('Failed to fetch')).toBe(true) + expect(isNetworkIssue('net::ERR_NETWORK_CHANGED')).toBe(true) + expect(isNetworkIssue(new UploadNetworkError('Network error while signing request'))).toBe( + true, + ) + expect( + isNetworkIssue(new UploadHttpError('Bad Gateway', { status: 502, isTransient: true })), + ).toBe(true) + expect( + isNetworkIssue( + new UploadHttpError('Service Unavailable', { status: 503, isTransient: true }), + ), + ).toBe(true) + expect( + isNetworkIssue(new UploadHttpError('Gateway Timeout', { status: 504, isTransient: true })), + ).toBe(true) + expect( + isNetworkIssue(new UploadHttpError('Rate Limited', { status: 429, isTransient: true })), + ).toBe(true) + expect( + isNetworkIssue( + new UploadHttpError('Internal Server Error', { status: 500, isTransient: false }), + ), + ).toBe(false) + expect( + isNetworkIssue(new UploadHttpError('Forbidden', { status: 403, isTransient: false })), + ).toBe(false) + expect(isNetworkIssue({ status: 502 })).toBe(true) + expect(isNetworkIssue({ status: 500 })).toBe(false) expect(isNetworkIssue(new Error('NetworkError when attempting to fetch resource.'))).toBe( true, ) @@ -590,4 +642,218 @@ describe('Uploader', () => { addFileSpy.mockRestore() }) + + it('keeps upload active and retries when socket drops with TypeError: Failed to fetch while navigator.onLine is still true', async () => { + Object.defineProperty(navigator, 'onLine', { value: true, configurable: true }) + useUploadStore + .getState() + .startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'socket-drop.mp4', size: 100 }]) + + const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1') + const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never) + + const uploadPromise = uploadFilesWithUppy({ + files: [ + { id: 'temp-1', file: new File(['content'], 'socket-drop.mp4', { type: 'video/mp4' }) }, + ], + taskId: 'task-1', + teamId: 'team-1', + storageBackend: 's3', + createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/socket-drop.mp4' }], + }) + + await new Promise((r) => setTimeout(r, 10)) + const capturedUppy = addFileSpy.mock.contexts[0] as Uppy + const mockFile = { + id: 'f1', + name: 'socket-drop.mp4', + meta: { fileId: 'file-1' }, + } as unknown as UppyFile> + + // Socket drops: navigator.onLine is STILL TRUE at this exact moment + const fetchError = new TypeError('Failed to fetch') + capturedUppy.emit('upload-error', mockFile, fetchError) + // Uppy emits complete with stripped error string + capturedUppy.emit('complete', { + successful: [], + failed: [{ ...mockFile, error: 'Failed to fetch' }], + }) + + await new Promise((r) => setTimeout(r, 20)) + + // Must NOT be marked failed, activeUploads must retain file, no error toast + expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading') + expect(activeUploads.has('file-1')).toBe(true) + expect(toast.error).not.toHaveBeenCalled() + expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).not.toHaveBeenCalled() + + // Simulate recovery on retry + capturedUppy.emit('upload-success', mockFile, { status: 200 }) + capturedUppy.emit('complete', { successful: [mockFile], failed: [] }) + + await uploadPromise + + expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed') + expect(activeUploads.size).toBe(0) + + addFileSpy.mockRestore() + retryAllSpy.mockRestore() + }) + + it('retries continuously on 502 Bad Gateway and recovers on subsequent attempt', async () => { + Object.defineProperty(navigator, 'onLine', { value: true, configurable: true }) + useUploadStore + .getState() + .startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'gateway.mp4', size: 100 }]) + + const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1') + const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never) + + const uploadPromise = uploadFilesWithUppy({ + files: [{ id: 'temp-1', file: new File(['content'], 'gateway.mp4', { type: 'video/mp4' }) }], + taskId: 'task-1', + teamId: 'team-1', + storageBackend: 's3', + createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/gateway.mp4' }], + }) + + await new Promise((r) => setTimeout(r, 10)) + const capturedUppy = addFileSpy.mock.contexts[0] as Uppy + const mockFile = { + id: 'f1', + name: 'gateway.mp4', + meta: { fileId: 'file-1' }, + } as unknown as UppyFile> + + const gatewayError = new UploadHttpError('Failed to sign request: Bad Gateway', { + status: 502, + isTransient: true, + }) + capturedUppy.emit('upload-error', mockFile, gatewayError) + capturedUppy.emit('complete', { + successful: [], + failed: [{ ...mockFile, error: 'Failed to sign request: Bad Gateway' }], + }) + + await new Promise((r) => setTimeout(r, 20)) + + // Must NOT be marked failed, activeUploads must retain file, no error toast + expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading') + expect(activeUploads.has('file-1')).toBe(true) + expect(toast.error).not.toHaveBeenCalled() + + // Simulate recovery on retry + capturedUppy.emit('upload-success', mockFile, { status: 200 }) + capturedUppy.emit('complete', { successful: [mockFile], failed: [] }) + + await uploadPromise + + expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed') + expect(activeUploads.size).toBe(0) + + addFileSpy.mockRestore() + retryAllSpy.mockRestore() + }) + + it('fails permanently on 500 Internal Server Error without retrying', async () => { + Object.defineProperty(navigator, 'onLine', { value: true, configurable: true }) + useUploadStore + .getState() + .startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'server-err.mp4', size: 100 }]) + + const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1') + + const uploadPromise = uploadFilesWithUppy({ + files: [ + { id: 'temp-1', file: new File(['content'], 'server-err.mp4', { type: 'video/mp4' }) }, + ], + taskId: 'task-1', + teamId: 'team-1', + storageBackend: 's3', + createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/server-err.mp4' }], + }) + + await new Promise((r) => setTimeout(r, 10)) + const capturedUppy = addFileSpy.mock.contexts[0] as Uppy + const mockFile = { + id: 'f1', + name: 'server-err.mp4', + meta: { fileId: 'file-1' }, + } as unknown as UppyFile> + + const serverError = new UploadHttpError('Internal Server Error', { + status: 500, + isTransient: false, + }) + capturedUppy.emit('upload-error', mockFile, serverError) + capturedUppy.emit('complete', { + successful: [], + failed: [{ ...mockFile, error: 'Internal Server Error' }], + }) + + await uploadPromise + + expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({ + param: { teamId: 'team-1', taskId: 'task-1' }, + json: { fileId: 'file-1', errorMessage: 'Internal Server Error' }, + }) + expect(toast.error).toHaveBeenCalledWith( + m.failed_upload_file_with_name({ name: 'server-err.mp4' }), + ) + expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed') + expect(activeUploads.size).toBe(0) + + addFileSpy.mockRestore() + }) + + it('fails permanently on 500/403 status even while navigator.onLine is false', async () => { + Object.defineProperty(navigator, 'onLine', { value: false, configurable: true }) + useUploadStore + .getState() + .startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'offline-500.mp4', size: 100 }]) + + const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1') + + const uploadPromise = uploadFilesWithUppy({ + files: [ + { id: 'temp-1', file: new File(['content'], 'offline-500.mp4', { type: 'video/mp4' }) }, + ], + taskId: 'task-1', + teamId: 'team-1', + storageBackend: 's3', + createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/offline-500.mp4' }], + }) + + await new Promise((r) => setTimeout(r, 10)) + const capturedUppy = addFileSpy.mock.contexts[0] as Uppy + const mockFile = { + id: 'f1', + name: 'offline-500.mp4', + meta: { fileId: 'file-1' }, + } as unknown as UppyFile> + + const serverError = new UploadHttpError('Internal Server Error', { + status: 500, + isTransient: false, + }) + capturedUppy.emit('upload-error', mockFile, serverError) + capturedUppy.emit('complete', { + successful: [], + failed: [{ ...mockFile, error: 'Internal Server Error' }], + }) + + await uploadPromise + + expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({ + param: { teamId: 'team-1', taskId: 'task-1' }, + json: { fileId: 'file-1', errorMessage: 'Internal Server Error' }, + }) + expect(toast.error).toHaveBeenCalledWith( + m.failed_upload_file_with_name({ name: 'offline-500.mp4' }), + ) + expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed') + expect(activeUploads.size).toBe(0) + + addFileSpy.mockRestore() + }) }) diff --git a/packages/webui/lib/uploader.ts b/packages/webui/lib/uploader.ts index 5c708187..fb3427a8 100644 --- a/packages/webui/lib/uploader.ts +++ b/packages/webui/lib/uploader.ts @@ -70,14 +70,38 @@ const LOCALHOST_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]', '0.0. export const MAX_RETRY_DELAY_MS = 60000 export const BASE_RETRY_DELAY_MS = 1000 +export class UploadNetworkError extends Error { + readonly code = 'NETWORK_ERROR' as const + constructor(message: string, options?: ErrorOptions) { + super(message, options) + this.name = 'UploadNetworkError' + } +} + +export class UploadHttpError extends Error { + readonly code = 'HTTP_ERROR' as const + readonly status: number + readonly isTransient: boolean + constructor(message: string, options: { status: number; isTransient: boolean; cause?: unknown }) { + super(message, { cause: options.cause }) + this.name = 'UploadHttpError' + this.status = options.status + this.isTransient = options.isTransient + } +} + export function calculateBackoffDelay( attempts: number, maxDelayMs = MAX_RETRY_DELAY_MS, baseDelayMs = BASE_RETRY_DELAY_MS, ): number { - const exponentialDelay = Math.min(Math.pow(2, attempts) * baseDelayMs, maxDelayMs) + const effectiveBaseDelay = + (typeof window !== 'undefined' && + (window as unknown as Record)['__SHUMAI_UPLOAD_BASE_DELAY__']) || + baseDelayMs + const exponentialDelay = Math.min(Math.pow(2, attempts) * effectiveBaseDelay, maxDelayMs) const jitter = exponentialDelay * (Math.random() * 0.3 - 0.15) // ±15% - return Math.min(maxDelayMs, Math.max(baseDelayMs, Math.round(exponentialDelay + jitter))) + return Math.min(maxDelayMs, Math.max(effectiveBaseDelay, Math.round(exponentialDelay + jitter))) } export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean { @@ -94,25 +118,98 @@ export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean { } export function isNetworkIssue(error?: unknown): boolean { + if (!error) { + return typeof navigator !== 'undefined' && !navigator.onLine + } + + // 1. Structured typed errors + if (error instanceof UploadHttpError) { + return error.isTransient + } + if (error instanceof UploadNetworkError) { + return true + } + + const err = (typeof error === 'object' && error !== null ? error : {}) as Record + + // 2. HTTP status inspection (check first before generic offline or NetworkError flags, + // so non-retryable HTTP 500 / 4xx responses delivered as the browser transitions offline + // fail immediately instead of being treated as network drops) + const status = + typeof err.status === 'number' + ? err.status + : typeof (err.request as Record | undefined)?.status === 'number' + ? ((err.request as Record).status as number) + : undefined + + if (status !== undefined) { + if (status === 0 || status === 429 || status === 502 || status === 503 || status === 504) { + return true + } + // Explicitly non-retryable statuses (500 Internal Server Error, 4xx except 429) + if (status === 500 || (status >= 400 && status < 500)) { + return false + } + if (status >= 500 && status < 600) { + return true + } + } + + // 3. Fallback string matching for explicit non-retryable HTTP status messages + const rawMsg = + typeof error === 'string' ? error : typeof err.message === 'string' ? err.message : '' + if (rawMsg) { + const msg = rawMsg.toLowerCase() + // Explicit non-retryable status codes in string messages + if ( + msg.includes('status: 500') || + msg.includes('status: 400') || + msg.includes('status: 401') || + msg.includes('status: 403') || + msg.includes('status: 404') + ) { + return false + } + if ( + msg.includes('network error') || + msg.includes('failed to fetch') || + msg.includes('networkerror') || + msg.includes('network changed') || + msg.includes('err_network_changed') || + msg.includes('err_internet_disconnected') || + msg.includes('err_connection_') || + msg.includes('err_name_not_resolved') || + msg.includes('the network connection was lost') || + msg.includes('the internet connection appears to be offline') || + (msg.includes('load failed') && !msg.includes('upload failed')) || + msg.includes('timed out') || + msg.includes('timeout') || + msg.includes('bad gateway') || + msg.includes('gateway timeout') || + msg.includes('service unavailable') + ) { + return true + } + } + + // 4. Generic offline state (only when the error does not carry a non-retryable status code) if (typeof navigator !== 'undefined' && !navigator.onLine) { return true } - if (!error) return false - const err = error as Record - if (err.isNetworkError === true) return true - if (err.name === 'S3NetworkError' || err.name === 'NetworkError') return true - const msg = (typeof err.message === 'string' ? err.message : '').toLowerCase() - if ( - msg.includes('network error') || - msg.includes('failed to fetch') || - msg.includes('networkerror') || - (msg.includes('load failed') && !msg.includes('upload failed')) || - msg.includes('err_internet_disconnected') - ) { + + if (err.code === 'NETWORK' || err.code === 'NETWORK_ERROR') return true + if (err.name === 'S3NetworkError' || err.name === 'UploadNetworkError') { return true } - const req = err.request as Record | undefined - if (req && req.status === 0) return true + + // Standard fetch network failure + if (error instanceof TypeError) { + return true + } + + // If there was no HTTP status attached, check generic Uppy NetworkError flag + if (err.name === 'NetworkError' || err.isNetworkError === true) return true + return false } @@ -183,21 +280,34 @@ export async function uploadFilesWithUppy({ } } - const res = await fetch(`/api/teams/${teamId}/upload/sign`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - key: req.key, - method: req.method, - uploadId, - partNumber, - fileId, - }), - credentials: 'include', - }) + let res: Response + try { + res = await fetch(`/api/teams/${teamId}/upload/sign`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + key: req.key, + method: req.method, + uploadId, + partNumber, + fileId, + }), + credentials: 'include', + }) + } catch (err) { + if (err instanceof TypeError) { + throw new UploadNetworkError('Network error while signing request', { cause: err }) + } + throw err + } if (!res.ok) { - throw new Error(`Failed to sign request: ${res.statusText}`) + const isTransient = + res.status === 429 || res.status === 502 || res.status === 503 || res.status === 504 + throw new UploadHttpError(`Failed to sign request: ${res.statusText}`, { + status: res.status, + isTransient, + }) } const data = (await res.json()) as { url: string } @@ -235,6 +345,7 @@ export async function uploadFilesWithUppy({ const confirmationPromises = new Map>() const retryCounts = new Map() + const pendingNetworkRetries = new Set() let retryTimeout: ReturnType | null = null const triggerRetry = () => { @@ -290,6 +401,10 @@ export async function uploadFilesWithUppy({ uppy.on('upload-success', (file) => { if (!file) return const fileId = file.meta.fileId as string + if (fileId) { + pendingNetworkRetries.delete(fileId) + retryCounts.delete(fileId) + } activeUploads.delete(fileId) const confirmPromise = (async () => { @@ -371,6 +486,9 @@ export async function uploadFilesWithUppy({ console.warn( `[Uploader] Localhost storage endpoint mismatch detected for ${file.name}. Failing permanently without retry.`, ) + if (fileId) { + pendingNetworkRetries.delete(fileId) + } handlePermanentError(file, error) return } @@ -380,8 +498,11 @@ export async function uploadFilesWithUppy({ `[Uploader] Network error detected for ${file.name}. Keeping upload active to resume when connection is restored.`, ) + if (fileId) { + pendingNetworkRetries.add(fileId) + } + if (typeof navigator !== 'undefined' && navigator.onLine) { - const fileId = file.meta?.fileId as string | undefined const attempts = (fileId ? retryCounts.get(fileId) || 0 : 0) + 1 if (fileId) { retryCounts.set(fileId, attempts) @@ -396,6 +517,9 @@ export async function uploadFilesWithUppy({ return } + if (fileId) { + pendingNetworkRetries.delete(fileId) + } handlePermanentError(file, error) }) @@ -443,32 +567,24 @@ export async function uploadFilesWithUppy({ } const failedFiles = result?.failed || [] - const isOffline = typeof navigator !== 'undefined' && !navigator.onLine - const hasNetworkFailures = - isOffline || - failedFiles.some((f) => { - const err = (f as { error?: unknown })?.error - const fileMeta = (f as { meta?: Record })?.meta - const fId = fileMeta?.fileId as string | undefined - const uUrl = - (fileMeta?.url as string | undefined) || - (fId ? activeUploads.get(fId)?.url : undefined) - if (isLocalhostStorageMismatch(uUrl)) { - return false - } - return isNetworkIssue(err) - }) - - if (hasNetworkFailures) { - console.log( - '[Uploader] Batch completed with network issues; waiting for reconnection to resume...', - ) - return - } - for (const f of failedFiles) { const file = f as { name: string; meta: Record } - handlePermanentError(file, (f as { error?: unknown })?.error) + const fId = file.meta?.fileId as string | undefined + if (fId && !pendingNetworkRetries.has(fId)) { + const err = (f as { error?: unknown })?.error + if (isNetworkIssue(err)) { + pendingNetworkRetries.add(fId) + continue + } + handlePermanentError(file, err) + } + } + + if (pendingNetworkRetries.size > 0) { + console.log( + `[Uploader] Batch completed with ${pendingNetworkRetries.size} retrying file(s); waiting for reconnection or backoff to resume...`, + ) + return } if (activeUploads.size === 0) {