feat(upload): state-based retry tracking and resilient gateway error handling (#509)

* feat(upload): state-based retry tracking and resilient gateway error handling

* fix(upload): check non-retryable statuses before offline state in isNetworkIssue
This commit is contained in:
Yiling-J
2026-10-31 10:37:35 +08:00
committed by GitHub
parent 9529c63c0a
commit 77bd1422d8
6 changed files with 727 additions and 67 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Added
- **upload**: Improve upload resilience during network drops and device sleep by adding continuous exponential backoff retries and extending upload link expiration to 12 hours
- **upload**: Improve upload resilience during network drops, device sleep, and transient gateway errors by adding state-based exponential backoff retries and extending upload link expiration to 12 hours
### Fixed
@@ -358,7 +358,7 @@ describe('MobileFileBrowser', () => {
window.XMLHttpRequest = originalXhr
})
it('fails upload and confirms with error on network failure', async () => {
it('keeps upload active and does not fail task on network failure', async () => {
const originalXhr = window.XMLHttpRequest
const patchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({}) })
@@ -409,17 +409,13 @@ describe('MobileFileBrowser', () => {
fireEvent.change(fileInput, { target: { files: [testFile] } })
await vi.waitFor(() => {
expect(patchMock).toHaveBeenCalledWith(
expect.objectContaining({
param: { teamId: 'team-1', taskId: 'task-3' },
json: expect.objectContaining({
fileId: 'server-file-3',
errorMessage: expect.any(String),
}),
}),
)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((client.api.teams[':teamId'].upload.tasks as any)['$post']).toHaveBeenCalled()
})
// Network failures are retried automatically with backoff and do not fail task on server
expect(patchMock).not.toHaveBeenCalled()
window.XMLHttpRequest = originalXhr
})
+91 -2
View File
@@ -1,10 +1,13 @@
import { type ReactElement, useCallback, useRef, useState } from 'react'
import { type ReactElement, useCallback, useEffect, useRef, useState } from 'react'
import ReactDOM from 'react-dom/client'
import type { MediaController } from '@/ui/components/viewers/types'
import VideoPlayer from '@/ui/components/viewers/video/video-viewer'
import ModelViewer from '@/ui/components/viewers/model/model-viewer'
import { calculateFrameCenterTime } from '@/ui/components/viewers/video/utils'
import { useUiStore } from '@/ui/stores/ui'
import { uploadFilesWithUppy } from '@/ui/lib/uploader'
import { useUploadStore } from '@/ui/stores/upload'
import { Toaster } from 'sonner'
import {
SAMPLE_FRAME_RATE,
sampleVideoAsset,
@@ -185,9 +188,95 @@ function Harness(): ReactElement {
)
}
function UploaderHarness(): ReactElement {
const [status, setStatus] = useState<'idle' | 'uploading' | 'completed' | 'failed'>('idle')
const [errorMessage, setErrorMessage] = useState<string | null>(null)
const taskState = useUploadStore((state) => state.tasks['e2e-task-1'])
useEffect(() => {
;(window as unknown as Record<string, number>)['__SHUMAI_UPLOAD_BASE_DELAY__'] = 100
}, [])
const startUpload = async (file: File) => {
setStatus('uploading')
setErrorMessage(null)
const taskId = 'e2e-task-1'
const teamId = 'e2e-team-1'
const fileId = 'e2e-file-1'
useUploadStore
.getState()
.startTask(taskId, 'e2e-task', [{ fileId, name: file.name, size: file.size }])
try {
await uploadFilesWithUppy({
files: [{ id: 'temp-1', file }],
taskId,
teamId,
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: fileId, key: `files/${file.name}` }],
})
setStatus('completed')
} catch (err) {
setStatus('failed')
setErrorMessage(err instanceof Error ? err.message : String(err))
}
}
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0]
if (file) {
void startUpload(file)
}
}
const handleSimulatedStart = () => {
const mockFile = new File(['mock content for test'], 'test-file.mp4', {
type: 'video/mp4',
})
void startUpload(mockFile)
}
const derivedStatus = taskState?.files['e2e-file-1']?.status ?? status
return (
<div style={{ padding: 24, fontFamily: 'sans-serif' }}>
<Toaster position="top-right" />
<h2>Uploader Resilience E2E Harness</h2>
<div style={{ marginBottom: 16 }}>
<input data-testid="uploader-file-input" type="file" onChange={handleFileChange} />
<button
data-testid="start-upload-btn"
onClick={handleSimulatedStart}
style={{ marginLeft: 8 }}
>
Start Simulated Upload
</button>
</div>
<div style={{ display: 'flex', gap: 16, alignItems: 'center' }}>
<div>
Status: <span data-testid="upload-status">{derivedStatus}</span>
</div>
<div>
Progress:{' '}
<span data-testid="upload-progress">{taskState?.files['e2e-file-1']?.loaded ?? 0}</span>
</div>
{errorMessage && <div data-testid="upload-error">{errorMessage}</div>}
</div>
</div>
)
}
const rootElement = document.getElementById('root')
if (!rootElement) {
throw new Error('Harness root element not found')
}
ReactDOM.createRoot(rootElement).render(<Harness />)
const currentView =
typeof window !== 'undefined' ? new URLSearchParams(window.location.search).get('view') : null
ReactDOM.createRoot(rootElement).render(
currentView === 'uploader' ? <UploaderHarness /> : <Harness />,
)
@@ -0,0 +1,193 @@
import { expect, test } from '@playwright/test'
test.describe('Uploader Resilience E2E Tests', () => {
test.beforeEach(async ({ page }) => {
// Default route mocks for the isolated harness
await page.route('**/api/teams/**/upload/tasks/**', (route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ ok: true }),
}),
)
})
test('keeps upload active on network drop and automatically resumes on reconnection', async ({
page,
}) => {
let releaseSign: () => void = () => {}
const signGate = new Promise<void>((resolve) => {
releaseSign = resolve
})
let signCalls = 0
await page.route('**/api/teams/**/upload/sign', async (route) => {
signCalls++
if (signCalls === 1) {
// Wait until offline event is dispatched before releasing first sign call
await signGate
}
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }),
})
})
await page.route('**/mock-upload', (route) =>
route.fulfill({
status: 200,
headers: { ['ETag']: '"test-etag"' },
}),
)
await page.goto('/?view=uploader')
await expect(page.getByTestId('upload-status')).toHaveText('idle')
// Start upload
await page.getByTestId('start-upload-btn').click()
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
// Simulate cutting the connection
await page.context().setOffline(true)
releaseSign()
// Status remains 'uploading' during offline state and no error toast appears
await page.waitForTimeout(500)
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
// Reconnect network
await page.context().setOffline(false)
// Upload automatically resumes and completes
await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 })
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
})
test('retries with backoff on in-flight socket drop and completes when network recovers', async ({
page,
}) => {
let signCalls = 0
await page.route('**/api/teams/**/upload/sign', (route) => {
signCalls++
if (signCalls <= 2) {
// In-flight connection reset / socket drop
route.abort('connectionreset')
} else {
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }),
})
}
})
await page.route('**/mock-upload', (route) =>
route.fulfill({
status: 200,
headers: { ['ETag']: '"test-etag"' },
}),
)
await page.goto('/?view=uploader')
await page.getByTestId('start-upload-btn').click()
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
// After backoff retries, succeeds on attempt 3
await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 })
expect(signCalls).toBeGreaterThanOrEqual(3)
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
})
test('retries continuously on 502 Bad Gateway and recovers once server returns 200', async ({
page,
}) => {
let signAttempts = 0
await page.route('**/api/teams/**/upload/sign', (route) => {
signAttempts++
if (signAttempts <= 2) {
// Return 502 Bad Gateway for the first 2 attempts
route.fulfill({
status: 502,
contentType: 'text/html',
body: '<html><body>502 Bad Gateway</body></html>',
})
} else {
// Server recovers on attempt 3
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }),
})
}
})
await page.route('**/mock-upload', (route) =>
route.fulfill({
status: 200,
headers: { ['ETag']: '"test-etag"' },
}),
)
await page.goto('/?view=uploader')
await page.getByTestId('start-upload-btn').click()
// Verify it stays uploading without failing on 502
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
// Once server recovers, upload completes
await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 })
expect(signAttempts).toBeGreaterThanOrEqual(3)
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
})
test('halts immediately without retrying on 500 Internal Server Error', async ({ page }) => {
let signAttempts = 0
await page.route('**/api/teams/**/upload/sign', (route) => {
signAttempts++
route.fulfill({
status: 500,
contentType: 'application/json',
body: JSON.stringify({ message: 'Internal Server Error' }),
})
})
await page.goto('/?view=uploader')
await page.getByTestId('start-upload-btn').click()
// Must fail immediately with error toast
await expect(page.getByTestId('upload-status')).toHaveText('failed', { timeout: 5_000 })
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toBeVisible({
timeout: 5_000,
})
// 0 retries (attempt count must be 1)
expect(signAttempts).toBe(1)
})
test('halts immediately without retrying on 403 Forbidden', async ({ page }) => {
let signAttempts = 0
await page.route('**/api/teams/**/upload/sign', (route) => {
signAttempts++
route.fulfill({
status: 403,
contentType: 'application/json',
body: JSON.stringify({ message: 'Forbidden' }),
})
})
await page.goto('/?view=uploader')
await page.getByTestId('start-upload-btn').click()
// Must fail immediately with error toast
await expect(page.getByTestId('upload-status')).toHaveText('failed', { timeout: 5_000 })
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toBeVisible({
timeout: 5_000,
})
// 0 retries (attempt count must be 1)
expect(signAttempts).toBe(1)
})
})
+266
View File
@@ -8,6 +8,8 @@ import {
calculateBackoffDelay,
MAX_RETRY_DELAY_MS,
BASE_RETRY_DELAY_MS,
UploadNetworkError,
UploadHttpError,
} from './uploader'
import { acquireWakeLock, releaseWakeLock } from './wake-lock'
import { useUploadStore } from '@/ui/stores/upload'
@@ -206,9 +208,59 @@ describe('Uploader', () => {
Object.defineProperty(navigator, 'onLine', { value: false, configurable: true })
expect(isNetworkIssue(new Error('Any error'))).toBe(true)
expect(isNetworkIssue()).toBe(true)
// Non-retryable statuses must return false even while offline
expect(
isNetworkIssue(
new UploadHttpError('Internal Server Error', { status: 500, isTransient: false }),
),
).toBe(false)
expect(
isNetworkIssue(new UploadHttpError('Forbidden', { status: 403, isTransient: false })),
).toBe(false)
expect(isNetworkIssue({ status: 500 })).toBe(false)
expect(isNetworkIssue({ status: 403 })).toBe(false)
expect(isNetworkIssue({ request: { status: 500 } })).toBe(false)
expect(isNetworkIssue({ request: { status: 403 } })).toBe(false)
expect(isNetworkIssue('upload failed with status: 500')).toBe(false)
expect(isNetworkIssue('upload failed with status: 403')).toBe(false)
// Transient statuses while offline return true
expect(
isNetworkIssue(new UploadHttpError('Bad Gateway', { status: 502, isTransient: true })),
).toBe(true)
expect(isNetworkIssue({ status: 502 })).toBe(true)
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
expect(isNetworkIssue(new Error('Failed to fetch'))).toBe(true)
expect(isNetworkIssue(new TypeError('Failed to fetch'))).toBe(true)
expect(isNetworkIssue('Failed to fetch')).toBe(true)
expect(isNetworkIssue('net::ERR_NETWORK_CHANGED')).toBe(true)
expect(isNetworkIssue(new UploadNetworkError('Network error while signing request'))).toBe(
true,
)
expect(
isNetworkIssue(new UploadHttpError('Bad Gateway', { status: 502, isTransient: true })),
).toBe(true)
expect(
isNetworkIssue(
new UploadHttpError('Service Unavailable', { status: 503, isTransient: true }),
),
).toBe(true)
expect(
isNetworkIssue(new UploadHttpError('Gateway Timeout', { status: 504, isTransient: true })),
).toBe(true)
expect(
isNetworkIssue(new UploadHttpError('Rate Limited', { status: 429, isTransient: true })),
).toBe(true)
expect(
isNetworkIssue(
new UploadHttpError('Internal Server Error', { status: 500, isTransient: false }),
),
).toBe(false)
expect(
isNetworkIssue(new UploadHttpError('Forbidden', { status: 403, isTransient: false })),
).toBe(false)
expect(isNetworkIssue({ status: 502 })).toBe(true)
expect(isNetworkIssue({ status: 500 })).toBe(false)
expect(isNetworkIssue(new Error('NetworkError when attempting to fetch resource.'))).toBe(
true,
)
@@ -590,4 +642,218 @@ describe('Uploader', () => {
addFileSpy.mockRestore()
})
it('keeps upload active and retries when socket drops with TypeError: Failed to fetch while navigator.onLine is still true', async () => {
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
useUploadStore
.getState()
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'socket-drop.mp4', size: 100 }])
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never)
const uploadPromise = uploadFilesWithUppy({
files: [
{ id: 'temp-1', file: new File(['content'], 'socket-drop.mp4', { type: 'video/mp4' }) },
],
taskId: 'task-1',
teamId: 'team-1',
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/socket-drop.mp4' }],
})
await new Promise((r) => setTimeout(r, 10))
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
const mockFile = {
id: 'f1',
name: 'socket-drop.mp4',
meta: { fileId: 'file-1' },
} as unknown as UppyFile<Meta, Record<string, never>>
// Socket drops: navigator.onLine is STILL TRUE at this exact moment
const fetchError = new TypeError('Failed to fetch')
capturedUppy.emit('upload-error', mockFile, fetchError)
// Uppy emits complete with stripped error string
capturedUppy.emit('complete', {
successful: [],
failed: [{ ...mockFile, error: 'Failed to fetch' }],
})
await new Promise((r) => setTimeout(r, 20))
// Must NOT be marked failed, activeUploads must retain file, no error toast
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading')
expect(activeUploads.has('file-1')).toBe(true)
expect(toast.error).not.toHaveBeenCalled()
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).not.toHaveBeenCalled()
// Simulate recovery on retry
capturedUppy.emit('upload-success', mockFile, { status: 200 })
capturedUppy.emit('complete', { successful: [mockFile], failed: [] })
await uploadPromise
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed')
expect(activeUploads.size).toBe(0)
addFileSpy.mockRestore()
retryAllSpy.mockRestore()
})
it('retries continuously on 502 Bad Gateway and recovers on subsequent attempt', async () => {
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
useUploadStore
.getState()
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'gateway.mp4', size: 100 }])
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never)
const uploadPromise = uploadFilesWithUppy({
files: [{ id: 'temp-1', file: new File(['content'], 'gateway.mp4', { type: 'video/mp4' }) }],
taskId: 'task-1',
teamId: 'team-1',
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/gateway.mp4' }],
})
await new Promise((r) => setTimeout(r, 10))
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
const mockFile = {
id: 'f1',
name: 'gateway.mp4',
meta: { fileId: 'file-1' },
} as unknown as UppyFile<Meta, Record<string, never>>
const gatewayError = new UploadHttpError('Failed to sign request: Bad Gateway', {
status: 502,
isTransient: true,
})
capturedUppy.emit('upload-error', mockFile, gatewayError)
capturedUppy.emit('complete', {
successful: [],
failed: [{ ...mockFile, error: 'Failed to sign request: Bad Gateway' }],
})
await new Promise((r) => setTimeout(r, 20))
// Must NOT be marked failed, activeUploads must retain file, no error toast
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading')
expect(activeUploads.has('file-1')).toBe(true)
expect(toast.error).not.toHaveBeenCalled()
// Simulate recovery on retry
capturedUppy.emit('upload-success', mockFile, { status: 200 })
capturedUppy.emit('complete', { successful: [mockFile], failed: [] })
await uploadPromise
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed')
expect(activeUploads.size).toBe(0)
addFileSpy.mockRestore()
retryAllSpy.mockRestore()
})
it('fails permanently on 500 Internal Server Error without retrying', async () => {
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
useUploadStore
.getState()
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'server-err.mp4', size: 100 }])
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
const uploadPromise = uploadFilesWithUppy({
files: [
{ id: 'temp-1', file: new File(['content'], 'server-err.mp4', { type: 'video/mp4' }) },
],
taskId: 'task-1',
teamId: 'team-1',
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/server-err.mp4' }],
})
await new Promise((r) => setTimeout(r, 10))
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
const mockFile = {
id: 'f1',
name: 'server-err.mp4',
meta: { fileId: 'file-1' },
} as unknown as UppyFile<Meta, Record<string, never>>
const serverError = new UploadHttpError('Internal Server Error', {
status: 500,
isTransient: false,
})
capturedUppy.emit('upload-error', mockFile, serverError)
capturedUppy.emit('complete', {
successful: [],
failed: [{ ...mockFile, error: 'Internal Server Error' }],
})
await uploadPromise
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({
param: { teamId: 'team-1', taskId: 'task-1' },
json: { fileId: 'file-1', errorMessage: 'Internal Server Error' },
})
expect(toast.error).toHaveBeenCalledWith(
m.failed_upload_file_with_name({ name: 'server-err.mp4' }),
)
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed')
expect(activeUploads.size).toBe(0)
addFileSpy.mockRestore()
})
it('fails permanently on 500/403 status even while navigator.onLine is false', async () => {
Object.defineProperty(navigator, 'onLine', { value: false, configurable: true })
useUploadStore
.getState()
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'offline-500.mp4', size: 100 }])
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
const uploadPromise = uploadFilesWithUppy({
files: [
{ id: 'temp-1', file: new File(['content'], 'offline-500.mp4', { type: 'video/mp4' }) },
],
taskId: 'task-1',
teamId: 'team-1',
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/offline-500.mp4' }],
})
await new Promise((r) => setTimeout(r, 10))
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
const mockFile = {
id: 'f1',
name: 'offline-500.mp4',
meta: { fileId: 'file-1' },
} as unknown as UppyFile<Meta, Record<string, never>>
const serverError = new UploadHttpError('Internal Server Error', {
status: 500,
isTransient: false,
})
capturedUppy.emit('upload-error', mockFile, serverError)
capturedUppy.emit('complete', {
successful: [],
failed: [{ ...mockFile, error: 'Internal Server Error' }],
})
await uploadPromise
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({
param: { teamId: 'team-1', taskId: 'task-1' },
json: { fileId: 'file-1', errorMessage: 'Internal Server Error' },
})
expect(toast.error).toHaveBeenCalledWith(
m.failed_upload_file_with_name({ name: 'offline-500.mp4' }),
)
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed')
expect(activeUploads.size).toBe(0)
addFileSpy.mockRestore()
})
})
+170 -54
View File
@@ -70,14 +70,38 @@ const LOCALHOST_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]', '0.0.
export const MAX_RETRY_DELAY_MS = 60000
export const BASE_RETRY_DELAY_MS = 1000
export class UploadNetworkError extends Error {
readonly code = 'NETWORK_ERROR' as const
constructor(message: string, options?: ErrorOptions) {
super(message, options)
this.name = 'UploadNetworkError'
}
}
export class UploadHttpError extends Error {
readonly code = 'HTTP_ERROR' as const
readonly status: number
readonly isTransient: boolean
constructor(message: string, options: { status: number; isTransient: boolean; cause?: unknown }) {
super(message, { cause: options.cause })
this.name = 'UploadHttpError'
this.status = options.status
this.isTransient = options.isTransient
}
}
export function calculateBackoffDelay(
attempts: number,
maxDelayMs = MAX_RETRY_DELAY_MS,
baseDelayMs = BASE_RETRY_DELAY_MS,
): number {
const exponentialDelay = Math.min(Math.pow(2, attempts) * baseDelayMs, maxDelayMs)
const effectiveBaseDelay =
(typeof window !== 'undefined' &&
(window as unknown as Record<string, number | undefined>)['__SHUMAI_UPLOAD_BASE_DELAY__']) ||
baseDelayMs
const exponentialDelay = Math.min(Math.pow(2, attempts) * effectiveBaseDelay, maxDelayMs)
const jitter = exponentialDelay * (Math.random() * 0.3 - 0.15) // ±15%
return Math.min(maxDelayMs, Math.max(baseDelayMs, Math.round(exponentialDelay + jitter)))
return Math.min(maxDelayMs, Math.max(effectiveBaseDelay, Math.round(exponentialDelay + jitter)))
}
export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean {
@@ -94,25 +118,98 @@ export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean {
}
export function isNetworkIssue(error?: unknown): boolean {
if (!error) {
return typeof navigator !== 'undefined' && !navigator.onLine
}
// 1. Structured typed errors
if (error instanceof UploadHttpError) {
return error.isTransient
}
if (error instanceof UploadNetworkError) {
return true
}
const err = (typeof error === 'object' && error !== null ? error : {}) as Record<string, unknown>
// 2. HTTP status inspection (check first before generic offline or NetworkError flags,
// so non-retryable HTTP 500 / 4xx responses delivered as the browser transitions offline
// fail immediately instead of being treated as network drops)
const status =
typeof err.status === 'number'
? err.status
: typeof (err.request as Record<string, unknown> | undefined)?.status === 'number'
? ((err.request as Record<string, unknown>).status as number)
: undefined
if (status !== undefined) {
if (status === 0 || status === 429 || status === 502 || status === 503 || status === 504) {
return true
}
// Explicitly non-retryable statuses (500 Internal Server Error, 4xx except 429)
if (status === 500 || (status >= 400 && status < 500)) {
return false
}
if (status >= 500 && status < 600) {
return true
}
}
// 3. Fallback string matching for explicit non-retryable HTTP status messages
const rawMsg =
typeof error === 'string' ? error : typeof err.message === 'string' ? err.message : ''
if (rawMsg) {
const msg = rawMsg.toLowerCase()
// Explicit non-retryable status codes in string messages
if (
msg.includes('status: 500') ||
msg.includes('status: 400') ||
msg.includes('status: 401') ||
msg.includes('status: 403') ||
msg.includes('status: 404')
) {
return false
}
if (
msg.includes('network error') ||
msg.includes('failed to fetch') ||
msg.includes('networkerror') ||
msg.includes('network changed') ||
msg.includes('err_network_changed') ||
msg.includes('err_internet_disconnected') ||
msg.includes('err_connection_') ||
msg.includes('err_name_not_resolved') ||
msg.includes('the network connection was lost') ||
msg.includes('the internet connection appears to be offline') ||
(msg.includes('load failed') && !msg.includes('upload failed')) ||
msg.includes('timed out') ||
msg.includes('timeout') ||
msg.includes('bad gateway') ||
msg.includes('gateway timeout') ||
msg.includes('service unavailable')
) {
return true
}
}
// 4. Generic offline state (only when the error does not carry a non-retryable status code)
if (typeof navigator !== 'undefined' && !navigator.onLine) {
return true
}
if (!error) return false
const err = error as Record<string, unknown>
if (err.isNetworkError === true) return true
if (err.name === 'S3NetworkError' || err.name === 'NetworkError') return true
const msg = (typeof err.message === 'string' ? err.message : '').toLowerCase()
if (
msg.includes('network error') ||
msg.includes('failed to fetch') ||
msg.includes('networkerror') ||
(msg.includes('load failed') && !msg.includes('upload failed')) ||
msg.includes('err_internet_disconnected')
) {
if (err.code === 'NETWORK' || err.code === 'NETWORK_ERROR') return true
if (err.name === 'S3NetworkError' || err.name === 'UploadNetworkError') {
return true
}
const req = err.request as Record<string, unknown> | undefined
if (req && req.status === 0) return true
// Standard fetch network failure
if (error instanceof TypeError) {
return true
}
// If there was no HTTP status attached, check generic Uppy NetworkError flag
if (err.name === 'NetworkError' || err.isNetworkError === true) return true
return false
}
@@ -183,21 +280,34 @@ export async function uploadFilesWithUppy({
}
}
const res = await fetch(`/api/teams/${teamId}/upload/sign`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
key: req.key,
method: req.method,
uploadId,
partNumber,
fileId,
}),
credentials: 'include',
})
let res: Response
try {
res = await fetch(`/api/teams/${teamId}/upload/sign`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
key: req.key,
method: req.method,
uploadId,
partNumber,
fileId,
}),
credentials: 'include',
})
} catch (err) {
if (err instanceof TypeError) {
throw new UploadNetworkError('Network error while signing request', { cause: err })
}
throw err
}
if (!res.ok) {
throw new Error(`Failed to sign request: ${res.statusText}`)
const isTransient =
res.status === 429 || res.status === 502 || res.status === 503 || res.status === 504
throw new UploadHttpError(`Failed to sign request: ${res.statusText}`, {
status: res.status,
isTransient,
})
}
const data = (await res.json()) as { url: string }
@@ -235,6 +345,7 @@ export async function uploadFilesWithUppy({
const confirmationPromises = new Map<string, Promise<void>>()
const retryCounts = new Map<string, number>()
const pendingNetworkRetries = new Set<string>()
let retryTimeout: ReturnType<typeof setTimeout> | null = null
const triggerRetry = () => {
@@ -290,6 +401,10 @@ export async function uploadFilesWithUppy({
uppy.on('upload-success', (file) => {
if (!file) return
const fileId = file.meta.fileId as string
if (fileId) {
pendingNetworkRetries.delete(fileId)
retryCounts.delete(fileId)
}
activeUploads.delete(fileId)
const confirmPromise = (async () => {
@@ -371,6 +486,9 @@ export async function uploadFilesWithUppy({
console.warn(
`[Uploader] Localhost storage endpoint mismatch detected for ${file.name}. Failing permanently without retry.`,
)
if (fileId) {
pendingNetworkRetries.delete(fileId)
}
handlePermanentError(file, error)
return
}
@@ -380,8 +498,11 @@ export async function uploadFilesWithUppy({
`[Uploader] Network error detected for ${file.name}. Keeping upload active to resume when connection is restored.`,
)
if (fileId) {
pendingNetworkRetries.add(fileId)
}
if (typeof navigator !== 'undefined' && navigator.onLine) {
const fileId = file.meta?.fileId as string | undefined
const attempts = (fileId ? retryCounts.get(fileId) || 0 : 0) + 1
if (fileId) {
retryCounts.set(fileId, attempts)
@@ -396,6 +517,9 @@ export async function uploadFilesWithUppy({
return
}
if (fileId) {
pendingNetworkRetries.delete(fileId)
}
handlePermanentError(file, error)
})
@@ -443,32 +567,24 @@ export async function uploadFilesWithUppy({
}
const failedFiles = result?.failed || []
const isOffline = typeof navigator !== 'undefined' && !navigator.onLine
const hasNetworkFailures =
isOffline ||
failedFiles.some((f) => {
const err = (f as { error?: unknown })?.error
const fileMeta = (f as { meta?: Record<string, unknown> })?.meta
const fId = fileMeta?.fileId as string | undefined
const uUrl =
(fileMeta?.url as string | undefined) ||
(fId ? activeUploads.get(fId)?.url : undefined)
if (isLocalhostStorageMismatch(uUrl)) {
return false
}
return isNetworkIssue(err)
})
if (hasNetworkFailures) {
console.log(
'[Uploader] Batch completed with network issues; waiting for reconnection to resume...',
)
return
}
for (const f of failedFiles) {
const file = f as { name: string; meta: Record<string, unknown> }
handlePermanentError(file, (f as { error?: unknown })?.error)
const fId = file.meta?.fileId as string | undefined
if (fId && !pendingNetworkRetries.has(fId)) {
const err = (f as { error?: unknown })?.error
if (isNetworkIssue(err)) {
pendingNetworkRetries.add(fId)
continue
}
handlePermanentError(file, err)
}
}
if (pendingNetworkRetries.size > 0) {
console.log(
`[Uploader] Batch completed with ${pendingNetworkRetries.size} retrying file(s); waiting for reconnection or backoff to resume...`,
)
return
}
if (activeUploads.size === 0) {