mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-19 14:16:29 +00:00
feat(upload): state-based retry tracking and resilient gateway error handling (#509)
* feat(upload): state-based retry tracking and resilient gateway error handling * fix(upload): check non-retryable statuses before offline state in isNetworkIssue
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Added
|
||||
|
||||
- **upload**: Improve upload resilience during network drops and device sleep by adding continuous exponential backoff retries and extending upload link expiration to 12 hours
|
||||
- **upload**: Improve upload resilience during network drops, device sleep, and transient gateway errors by adding state-based exponential backoff retries and extending upload link expiration to 12 hours
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@@ -358,7 +358,7 @@ describe('MobileFileBrowser', () => {
|
||||
window.XMLHttpRequest = originalXhr
|
||||
})
|
||||
|
||||
it('fails upload and confirms with error on network failure', async () => {
|
||||
it('keeps upload active and does not fail task on network failure', async () => {
|
||||
const originalXhr = window.XMLHttpRequest
|
||||
const patchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({}) })
|
||||
|
||||
@@ -409,17 +409,13 @@ describe('MobileFileBrowser', () => {
|
||||
fireEvent.change(fileInput, { target: { files: [testFile] } })
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(patchMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
param: { teamId: 'team-1', taskId: 'task-3' },
|
||||
json: expect.objectContaining({
|
||||
fileId: 'server-file-3',
|
||||
errorMessage: expect.any(String),
|
||||
}),
|
||||
}),
|
||||
)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((client.api.teams[':teamId'].upload.tasks as any)['$post']).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Network failures are retried automatically with backoff and do not fail task on server
|
||||
expect(patchMock).not.toHaveBeenCalled()
|
||||
|
||||
window.XMLHttpRequest = originalXhr
|
||||
})
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { type ReactElement, useCallback, useRef, useState } from 'react'
|
||||
import { type ReactElement, useCallback, useEffect, useRef, useState } from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import type { MediaController } from '@/ui/components/viewers/types'
|
||||
import VideoPlayer from '@/ui/components/viewers/video/video-viewer'
|
||||
import ModelViewer from '@/ui/components/viewers/model/model-viewer'
|
||||
import { calculateFrameCenterTime } from '@/ui/components/viewers/video/utils'
|
||||
import { useUiStore } from '@/ui/stores/ui'
|
||||
import { uploadFilesWithUppy } from '@/ui/lib/uploader'
|
||||
import { useUploadStore } from '@/ui/stores/upload'
|
||||
import { Toaster } from 'sonner'
|
||||
import {
|
||||
SAMPLE_FRAME_RATE,
|
||||
sampleVideoAsset,
|
||||
@@ -185,9 +188,95 @@ function Harness(): ReactElement {
|
||||
)
|
||||
}
|
||||
|
||||
function UploaderHarness(): ReactElement {
|
||||
const [status, setStatus] = useState<'idle' | 'uploading' | 'completed' | 'failed'>('idle')
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null)
|
||||
const taskState = useUploadStore((state) => state.tasks['e2e-task-1'])
|
||||
|
||||
useEffect(() => {
|
||||
;(window as unknown as Record<string, number>)['__SHUMAI_UPLOAD_BASE_DELAY__'] = 100
|
||||
}, [])
|
||||
|
||||
const startUpload = async (file: File) => {
|
||||
setStatus('uploading')
|
||||
setErrorMessage(null)
|
||||
|
||||
const taskId = 'e2e-task-1'
|
||||
const teamId = 'e2e-team-1'
|
||||
const fileId = 'e2e-file-1'
|
||||
|
||||
useUploadStore
|
||||
.getState()
|
||||
.startTask(taskId, 'e2e-task', [{ fileId, name: file.name, size: file.size }])
|
||||
|
||||
try {
|
||||
await uploadFilesWithUppy({
|
||||
files: [{ id: 'temp-1', file }],
|
||||
taskId,
|
||||
teamId,
|
||||
storageBackend: 's3',
|
||||
createdAssets: [{ tempId: 'temp-1', assetId: fileId, key: `files/${file.name}` }],
|
||||
})
|
||||
setStatus('completed')
|
||||
} catch (err) {
|
||||
setStatus('failed')
|
||||
setErrorMessage(err instanceof Error ? err.message : String(err))
|
||||
}
|
||||
}
|
||||
|
||||
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = e.target.files?.[0]
|
||||
if (file) {
|
||||
void startUpload(file)
|
||||
}
|
||||
}
|
||||
|
||||
const handleSimulatedStart = () => {
|
||||
const mockFile = new File(['mock content for test'], 'test-file.mp4', {
|
||||
type: 'video/mp4',
|
||||
})
|
||||
void startUpload(mockFile)
|
||||
}
|
||||
|
||||
const derivedStatus = taskState?.files['e2e-file-1']?.status ?? status
|
||||
|
||||
return (
|
||||
<div style={{ padding: 24, fontFamily: 'sans-serif' }}>
|
||||
<Toaster position="top-right" />
|
||||
<h2>Uploader Resilience E2E Harness</h2>
|
||||
<div style={{ marginBottom: 16 }}>
|
||||
<input data-testid="uploader-file-input" type="file" onChange={handleFileChange} />
|
||||
<button
|
||||
data-testid="start-upload-btn"
|
||||
onClick={handleSimulatedStart}
|
||||
style={{ marginLeft: 8 }}
|
||||
>
|
||||
Start Simulated Upload
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', gap: 16, alignItems: 'center' }}>
|
||||
<div>
|
||||
Status: <span data-testid="upload-status">{derivedStatus}</span>
|
||||
</div>
|
||||
<div>
|
||||
Progress:{' '}
|
||||
<span data-testid="upload-progress">{taskState?.files['e2e-file-1']?.loaded ?? 0}</span>
|
||||
</div>
|
||||
{errorMessage && <div data-testid="upload-error">{errorMessage}</div>}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const rootElement = document.getElementById('root')
|
||||
if (!rootElement) {
|
||||
throw new Error('Harness root element not found')
|
||||
}
|
||||
|
||||
ReactDOM.createRoot(rootElement).render(<Harness />)
|
||||
const currentView =
|
||||
typeof window !== 'undefined' ? new URLSearchParams(window.location.search).get('view') : null
|
||||
|
||||
ReactDOM.createRoot(rootElement).render(
|
||||
currentView === 'uploader' ? <UploaderHarness /> : <Harness />,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { expect, test } from '@playwright/test'
|
||||
|
||||
test.describe('Uploader Resilience E2E Tests', () => {
|
||||
test.beforeEach(async ({ page }) => {
|
||||
// Default route mocks for the isolated harness
|
||||
await page.route('**/api/teams/**/upload/tasks/**', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ ok: true }),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test('keeps upload active on network drop and automatically resumes on reconnection', async ({
|
||||
page,
|
||||
}) => {
|
||||
let releaseSign: () => void = () => {}
|
||||
const signGate = new Promise<void>((resolve) => {
|
||||
releaseSign = resolve
|
||||
})
|
||||
|
||||
let signCalls = 0
|
||||
await page.route('**/api/teams/**/upload/sign', async (route) => {
|
||||
signCalls++
|
||||
if (signCalls === 1) {
|
||||
// Wait until offline event is dispatched before releasing first sign call
|
||||
await signGate
|
||||
}
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }),
|
||||
})
|
||||
})
|
||||
|
||||
await page.route('**/mock-upload', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
headers: { ['ETag']: '"test-etag"' },
|
||||
}),
|
||||
)
|
||||
|
||||
await page.goto('/?view=uploader')
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('idle')
|
||||
|
||||
// Start upload
|
||||
await page.getByTestId('start-upload-btn').click()
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
|
||||
|
||||
// Simulate cutting the connection
|
||||
await page.context().setOffline(true)
|
||||
releaseSign()
|
||||
|
||||
// Status remains 'uploading' during offline state and no error toast appears
|
||||
await page.waitForTimeout(500)
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
|
||||
|
||||
// Reconnect network
|
||||
await page.context().setOffline(false)
|
||||
|
||||
// Upload automatically resumes and completes
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 })
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
|
||||
})
|
||||
|
||||
test('retries with backoff on in-flight socket drop and completes when network recovers', async ({
|
||||
page,
|
||||
}) => {
|
||||
let signCalls = 0
|
||||
await page.route('**/api/teams/**/upload/sign', (route) => {
|
||||
signCalls++
|
||||
if (signCalls <= 2) {
|
||||
// In-flight connection reset / socket drop
|
||||
route.abort('connectionreset')
|
||||
} else {
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
await page.route('**/mock-upload', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
headers: { ['ETag']: '"test-etag"' },
|
||||
}),
|
||||
)
|
||||
|
||||
await page.goto('/?view=uploader')
|
||||
await page.getByTestId('start-upload-btn').click()
|
||||
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
|
||||
|
||||
// After backoff retries, succeeds on attempt 3
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 })
|
||||
expect(signCalls).toBeGreaterThanOrEqual(3)
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
|
||||
})
|
||||
|
||||
test('retries continuously on 502 Bad Gateway and recovers once server returns 200', async ({
|
||||
page,
|
||||
}) => {
|
||||
let signAttempts = 0
|
||||
await page.route('**/api/teams/**/upload/sign', (route) => {
|
||||
signAttempts++
|
||||
if (signAttempts <= 2) {
|
||||
// Return 502 Bad Gateway for the first 2 attempts
|
||||
route.fulfill({
|
||||
status: 502,
|
||||
contentType: 'text/html',
|
||||
body: '<html><body>502 Bad Gateway</body></html>',
|
||||
})
|
||||
} else {
|
||||
// Server recovers on attempt 3
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ url: 'http://localhost:5199/mock-upload' }),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
await page.route('**/mock-upload', (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
headers: { ['ETag']: '"test-etag"' },
|
||||
}),
|
||||
)
|
||||
|
||||
await page.goto('/?view=uploader')
|
||||
await page.getByTestId('start-upload-btn').click()
|
||||
|
||||
// Verify it stays uploading without failing on 502
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('uploading')
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
|
||||
|
||||
// Once server recovers, upload completes
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('completed', { timeout: 15_000 })
|
||||
expect(signAttempts).toBeGreaterThanOrEqual(3)
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toHaveCount(0)
|
||||
})
|
||||
|
||||
test('halts immediately without retrying on 500 Internal Server Error', async ({ page }) => {
|
||||
let signAttempts = 0
|
||||
await page.route('**/api/teams/**/upload/sign', (route) => {
|
||||
signAttempts++
|
||||
route.fulfill({
|
||||
status: 500,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ message: 'Internal Server Error' }),
|
||||
})
|
||||
})
|
||||
|
||||
await page.goto('/?view=uploader')
|
||||
await page.getByTestId('start-upload-btn').click()
|
||||
|
||||
// Must fail immediately with error toast
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('failed', { timeout: 5_000 })
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toBeVisible({
|
||||
timeout: 5_000,
|
||||
})
|
||||
// 0 retries (attempt count must be 1)
|
||||
expect(signAttempts).toBe(1)
|
||||
})
|
||||
|
||||
test('halts immediately without retrying on 403 Forbidden', async ({ page }) => {
|
||||
let signAttempts = 0
|
||||
await page.route('**/api/teams/**/upload/sign', (route) => {
|
||||
signAttempts++
|
||||
route.fulfill({
|
||||
status: 403,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ message: 'Forbidden' }),
|
||||
})
|
||||
})
|
||||
|
||||
await page.goto('/?view=uploader')
|
||||
await page.getByTestId('start-upload-btn').click()
|
||||
|
||||
// Must fail immediately with error toast
|
||||
await expect(page.getByTestId('upload-status')).toHaveText('failed', { timeout: 5_000 })
|
||||
await expect(page.locator('[data-sonner-toast][data-type="error"]')).toBeVisible({
|
||||
timeout: 5_000,
|
||||
})
|
||||
// 0 retries (attempt count must be 1)
|
||||
expect(signAttempts).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
calculateBackoffDelay,
|
||||
MAX_RETRY_DELAY_MS,
|
||||
BASE_RETRY_DELAY_MS,
|
||||
UploadNetworkError,
|
||||
UploadHttpError,
|
||||
} from './uploader'
|
||||
import { acquireWakeLock, releaseWakeLock } from './wake-lock'
|
||||
import { useUploadStore } from '@/ui/stores/upload'
|
||||
@@ -206,9 +208,59 @@ describe('Uploader', () => {
|
||||
Object.defineProperty(navigator, 'onLine', { value: false, configurable: true })
|
||||
expect(isNetworkIssue(new Error('Any error'))).toBe(true)
|
||||
expect(isNetworkIssue()).toBe(true)
|
||||
// Non-retryable statuses must return false even while offline
|
||||
expect(
|
||||
isNetworkIssue(
|
||||
new UploadHttpError('Internal Server Error', { status: 500, isTransient: false }),
|
||||
),
|
||||
).toBe(false)
|
||||
expect(
|
||||
isNetworkIssue(new UploadHttpError('Forbidden', { status: 403, isTransient: false })),
|
||||
).toBe(false)
|
||||
expect(isNetworkIssue({ status: 500 })).toBe(false)
|
||||
expect(isNetworkIssue({ status: 403 })).toBe(false)
|
||||
expect(isNetworkIssue({ request: { status: 500 } })).toBe(false)
|
||||
expect(isNetworkIssue({ request: { status: 403 } })).toBe(false)
|
||||
expect(isNetworkIssue('upload failed with status: 500')).toBe(false)
|
||||
expect(isNetworkIssue('upload failed with status: 403')).toBe(false)
|
||||
// Transient statuses while offline return true
|
||||
expect(
|
||||
isNetworkIssue(new UploadHttpError('Bad Gateway', { status: 502, isTransient: true })),
|
||||
).toBe(true)
|
||||
expect(isNetworkIssue({ status: 502 })).toBe(true)
|
||||
|
||||
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
|
||||
expect(isNetworkIssue(new Error('Failed to fetch'))).toBe(true)
|
||||
expect(isNetworkIssue(new TypeError('Failed to fetch'))).toBe(true)
|
||||
expect(isNetworkIssue('Failed to fetch')).toBe(true)
|
||||
expect(isNetworkIssue('net::ERR_NETWORK_CHANGED')).toBe(true)
|
||||
expect(isNetworkIssue(new UploadNetworkError('Network error while signing request'))).toBe(
|
||||
true,
|
||||
)
|
||||
expect(
|
||||
isNetworkIssue(new UploadHttpError('Bad Gateway', { status: 502, isTransient: true })),
|
||||
).toBe(true)
|
||||
expect(
|
||||
isNetworkIssue(
|
||||
new UploadHttpError('Service Unavailable', { status: 503, isTransient: true }),
|
||||
),
|
||||
).toBe(true)
|
||||
expect(
|
||||
isNetworkIssue(new UploadHttpError('Gateway Timeout', { status: 504, isTransient: true })),
|
||||
).toBe(true)
|
||||
expect(
|
||||
isNetworkIssue(new UploadHttpError('Rate Limited', { status: 429, isTransient: true })),
|
||||
).toBe(true)
|
||||
expect(
|
||||
isNetworkIssue(
|
||||
new UploadHttpError('Internal Server Error', { status: 500, isTransient: false }),
|
||||
),
|
||||
).toBe(false)
|
||||
expect(
|
||||
isNetworkIssue(new UploadHttpError('Forbidden', { status: 403, isTransient: false })),
|
||||
).toBe(false)
|
||||
expect(isNetworkIssue({ status: 502 })).toBe(true)
|
||||
expect(isNetworkIssue({ status: 500 })).toBe(false)
|
||||
expect(isNetworkIssue(new Error('NetworkError when attempting to fetch resource.'))).toBe(
|
||||
true,
|
||||
)
|
||||
@@ -590,4 +642,218 @@ describe('Uploader', () => {
|
||||
|
||||
addFileSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('keeps upload active and retries when socket drops with TypeError: Failed to fetch while navigator.onLine is still true', async () => {
|
||||
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
|
||||
useUploadStore
|
||||
.getState()
|
||||
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'socket-drop.mp4', size: 100 }])
|
||||
|
||||
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
|
||||
const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never)
|
||||
|
||||
const uploadPromise = uploadFilesWithUppy({
|
||||
files: [
|
||||
{ id: 'temp-1', file: new File(['content'], 'socket-drop.mp4', { type: 'video/mp4' }) },
|
||||
],
|
||||
taskId: 'task-1',
|
||||
teamId: 'team-1',
|
||||
storageBackend: 's3',
|
||||
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/socket-drop.mp4' }],
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 10))
|
||||
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
|
||||
const mockFile = {
|
||||
id: 'f1',
|
||||
name: 'socket-drop.mp4',
|
||||
meta: { fileId: 'file-1' },
|
||||
} as unknown as UppyFile<Meta, Record<string, never>>
|
||||
|
||||
// Socket drops: navigator.onLine is STILL TRUE at this exact moment
|
||||
const fetchError = new TypeError('Failed to fetch')
|
||||
capturedUppy.emit('upload-error', mockFile, fetchError)
|
||||
// Uppy emits complete with stripped error string
|
||||
capturedUppy.emit('complete', {
|
||||
successful: [],
|
||||
failed: [{ ...mockFile, error: 'Failed to fetch' }],
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 20))
|
||||
|
||||
// Must NOT be marked failed, activeUploads must retain file, no error toast
|
||||
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading')
|
||||
expect(activeUploads.has('file-1')).toBe(true)
|
||||
expect(toast.error).not.toHaveBeenCalled()
|
||||
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).not.toHaveBeenCalled()
|
||||
|
||||
// Simulate recovery on retry
|
||||
capturedUppy.emit('upload-success', mockFile, { status: 200 })
|
||||
capturedUppy.emit('complete', { successful: [mockFile], failed: [] })
|
||||
|
||||
await uploadPromise
|
||||
|
||||
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed')
|
||||
expect(activeUploads.size).toBe(0)
|
||||
|
||||
addFileSpy.mockRestore()
|
||||
retryAllSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('retries continuously on 502 Bad Gateway and recovers on subsequent attempt', async () => {
|
||||
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
|
||||
useUploadStore
|
||||
.getState()
|
||||
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'gateway.mp4', size: 100 }])
|
||||
|
||||
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
|
||||
const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never)
|
||||
|
||||
const uploadPromise = uploadFilesWithUppy({
|
||||
files: [{ id: 'temp-1', file: new File(['content'], 'gateway.mp4', { type: 'video/mp4' }) }],
|
||||
taskId: 'task-1',
|
||||
teamId: 'team-1',
|
||||
storageBackend: 's3',
|
||||
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/gateway.mp4' }],
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 10))
|
||||
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
|
||||
const mockFile = {
|
||||
id: 'f1',
|
||||
name: 'gateway.mp4',
|
||||
meta: { fileId: 'file-1' },
|
||||
} as unknown as UppyFile<Meta, Record<string, never>>
|
||||
|
||||
const gatewayError = new UploadHttpError('Failed to sign request: Bad Gateway', {
|
||||
status: 502,
|
||||
isTransient: true,
|
||||
})
|
||||
capturedUppy.emit('upload-error', mockFile, gatewayError)
|
||||
capturedUppy.emit('complete', {
|
||||
successful: [],
|
||||
failed: [{ ...mockFile, error: 'Failed to sign request: Bad Gateway' }],
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 20))
|
||||
|
||||
// Must NOT be marked failed, activeUploads must retain file, no error toast
|
||||
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading')
|
||||
expect(activeUploads.has('file-1')).toBe(true)
|
||||
expect(toast.error).not.toHaveBeenCalled()
|
||||
|
||||
// Simulate recovery on retry
|
||||
capturedUppy.emit('upload-success', mockFile, { status: 200 })
|
||||
capturedUppy.emit('complete', { successful: [mockFile], failed: [] })
|
||||
|
||||
await uploadPromise
|
||||
|
||||
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed')
|
||||
expect(activeUploads.size).toBe(0)
|
||||
|
||||
addFileSpy.mockRestore()
|
||||
retryAllSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('fails permanently on 500 Internal Server Error without retrying', async () => {
|
||||
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
|
||||
useUploadStore
|
||||
.getState()
|
||||
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'server-err.mp4', size: 100 }])
|
||||
|
||||
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
|
||||
|
||||
const uploadPromise = uploadFilesWithUppy({
|
||||
files: [
|
||||
{ id: 'temp-1', file: new File(['content'], 'server-err.mp4', { type: 'video/mp4' }) },
|
||||
],
|
||||
taskId: 'task-1',
|
||||
teamId: 'team-1',
|
||||
storageBackend: 's3',
|
||||
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/server-err.mp4' }],
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 10))
|
||||
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
|
||||
const mockFile = {
|
||||
id: 'f1',
|
||||
name: 'server-err.mp4',
|
||||
meta: { fileId: 'file-1' },
|
||||
} as unknown as UppyFile<Meta, Record<string, never>>
|
||||
|
||||
const serverError = new UploadHttpError('Internal Server Error', {
|
||||
status: 500,
|
||||
isTransient: false,
|
||||
})
|
||||
capturedUppy.emit('upload-error', mockFile, serverError)
|
||||
capturedUppy.emit('complete', {
|
||||
successful: [],
|
||||
failed: [{ ...mockFile, error: 'Internal Server Error' }],
|
||||
})
|
||||
|
||||
await uploadPromise
|
||||
|
||||
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({
|
||||
param: { teamId: 'team-1', taskId: 'task-1' },
|
||||
json: { fileId: 'file-1', errorMessage: 'Internal Server Error' },
|
||||
})
|
||||
expect(toast.error).toHaveBeenCalledWith(
|
||||
m.failed_upload_file_with_name({ name: 'server-err.mp4' }),
|
||||
)
|
||||
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed')
|
||||
expect(activeUploads.size).toBe(0)
|
||||
|
||||
addFileSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('fails permanently on 500/403 status even while navigator.onLine is false', async () => {
|
||||
Object.defineProperty(navigator, 'onLine', { value: false, configurable: true })
|
||||
useUploadStore
|
||||
.getState()
|
||||
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'offline-500.mp4', size: 100 }])
|
||||
|
||||
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
|
||||
|
||||
const uploadPromise = uploadFilesWithUppy({
|
||||
files: [
|
||||
{ id: 'temp-1', file: new File(['content'], 'offline-500.mp4', { type: 'video/mp4' }) },
|
||||
],
|
||||
taskId: 'task-1',
|
||||
teamId: 'team-1',
|
||||
storageBackend: 's3',
|
||||
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/offline-500.mp4' }],
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 10))
|
||||
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
|
||||
const mockFile = {
|
||||
id: 'f1',
|
||||
name: 'offline-500.mp4',
|
||||
meta: { fileId: 'file-1' },
|
||||
} as unknown as UppyFile<Meta, Record<string, never>>
|
||||
|
||||
const serverError = new UploadHttpError('Internal Server Error', {
|
||||
status: 500,
|
||||
isTransient: false,
|
||||
})
|
||||
capturedUppy.emit('upload-error', mockFile, serverError)
|
||||
capturedUppy.emit('complete', {
|
||||
successful: [],
|
||||
failed: [{ ...mockFile, error: 'Internal Server Error' }],
|
||||
})
|
||||
|
||||
await uploadPromise
|
||||
|
||||
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({
|
||||
param: { teamId: 'team-1', taskId: 'task-1' },
|
||||
json: { fileId: 'file-1', errorMessage: 'Internal Server Error' },
|
||||
})
|
||||
expect(toast.error).toHaveBeenCalledWith(
|
||||
m.failed_upload_file_with_name({ name: 'offline-500.mp4' }),
|
||||
)
|
||||
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed')
|
||||
expect(activeUploads.size).toBe(0)
|
||||
|
||||
addFileSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
+170
-54
@@ -70,14 +70,38 @@ const LOCALHOST_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]', '0.0.
|
||||
export const MAX_RETRY_DELAY_MS = 60000
|
||||
export const BASE_RETRY_DELAY_MS = 1000
|
||||
|
||||
export class UploadNetworkError extends Error {
|
||||
readonly code = 'NETWORK_ERROR' as const
|
||||
constructor(message: string, options?: ErrorOptions) {
|
||||
super(message, options)
|
||||
this.name = 'UploadNetworkError'
|
||||
}
|
||||
}
|
||||
|
||||
export class UploadHttpError extends Error {
|
||||
readonly code = 'HTTP_ERROR' as const
|
||||
readonly status: number
|
||||
readonly isTransient: boolean
|
||||
constructor(message: string, options: { status: number; isTransient: boolean; cause?: unknown }) {
|
||||
super(message, { cause: options.cause })
|
||||
this.name = 'UploadHttpError'
|
||||
this.status = options.status
|
||||
this.isTransient = options.isTransient
|
||||
}
|
||||
}
|
||||
|
||||
export function calculateBackoffDelay(
|
||||
attempts: number,
|
||||
maxDelayMs = MAX_RETRY_DELAY_MS,
|
||||
baseDelayMs = BASE_RETRY_DELAY_MS,
|
||||
): number {
|
||||
const exponentialDelay = Math.min(Math.pow(2, attempts) * baseDelayMs, maxDelayMs)
|
||||
const effectiveBaseDelay =
|
||||
(typeof window !== 'undefined' &&
|
||||
(window as unknown as Record<string, number | undefined>)['__SHUMAI_UPLOAD_BASE_DELAY__']) ||
|
||||
baseDelayMs
|
||||
const exponentialDelay = Math.min(Math.pow(2, attempts) * effectiveBaseDelay, maxDelayMs)
|
||||
const jitter = exponentialDelay * (Math.random() * 0.3 - 0.15) // ±15%
|
||||
return Math.min(maxDelayMs, Math.max(baseDelayMs, Math.round(exponentialDelay + jitter)))
|
||||
return Math.min(maxDelayMs, Math.max(effectiveBaseDelay, Math.round(exponentialDelay + jitter)))
|
||||
}
|
||||
|
||||
export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean {
|
||||
@@ -94,25 +118,98 @@ export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean {
|
||||
}
|
||||
|
||||
export function isNetworkIssue(error?: unknown): boolean {
|
||||
if (!error) {
|
||||
return typeof navigator !== 'undefined' && !navigator.onLine
|
||||
}
|
||||
|
||||
// 1. Structured typed errors
|
||||
if (error instanceof UploadHttpError) {
|
||||
return error.isTransient
|
||||
}
|
||||
if (error instanceof UploadNetworkError) {
|
||||
return true
|
||||
}
|
||||
|
||||
const err = (typeof error === 'object' && error !== null ? error : {}) as Record<string, unknown>
|
||||
|
||||
// 2. HTTP status inspection (check first before generic offline or NetworkError flags,
|
||||
// so non-retryable HTTP 500 / 4xx responses delivered as the browser transitions offline
|
||||
// fail immediately instead of being treated as network drops)
|
||||
const status =
|
||||
typeof err.status === 'number'
|
||||
? err.status
|
||||
: typeof (err.request as Record<string, unknown> | undefined)?.status === 'number'
|
||||
? ((err.request as Record<string, unknown>).status as number)
|
||||
: undefined
|
||||
|
||||
if (status !== undefined) {
|
||||
if (status === 0 || status === 429 || status === 502 || status === 503 || status === 504) {
|
||||
return true
|
||||
}
|
||||
// Explicitly non-retryable statuses (500 Internal Server Error, 4xx except 429)
|
||||
if (status === 500 || (status >= 400 && status < 500)) {
|
||||
return false
|
||||
}
|
||||
if (status >= 500 && status < 600) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Fallback string matching for explicit non-retryable HTTP status messages
|
||||
const rawMsg =
|
||||
typeof error === 'string' ? error : typeof err.message === 'string' ? err.message : ''
|
||||
if (rawMsg) {
|
||||
const msg = rawMsg.toLowerCase()
|
||||
// Explicit non-retryable status codes in string messages
|
||||
if (
|
||||
msg.includes('status: 500') ||
|
||||
msg.includes('status: 400') ||
|
||||
msg.includes('status: 401') ||
|
||||
msg.includes('status: 403') ||
|
||||
msg.includes('status: 404')
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
msg.includes('network error') ||
|
||||
msg.includes('failed to fetch') ||
|
||||
msg.includes('networkerror') ||
|
||||
msg.includes('network changed') ||
|
||||
msg.includes('err_network_changed') ||
|
||||
msg.includes('err_internet_disconnected') ||
|
||||
msg.includes('err_connection_') ||
|
||||
msg.includes('err_name_not_resolved') ||
|
||||
msg.includes('the network connection was lost') ||
|
||||
msg.includes('the internet connection appears to be offline') ||
|
||||
(msg.includes('load failed') && !msg.includes('upload failed')) ||
|
||||
msg.includes('timed out') ||
|
||||
msg.includes('timeout') ||
|
||||
msg.includes('bad gateway') ||
|
||||
msg.includes('gateway timeout') ||
|
||||
msg.includes('service unavailable')
|
||||
) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Generic offline state (only when the error does not carry a non-retryable status code)
|
||||
if (typeof navigator !== 'undefined' && !navigator.onLine) {
|
||||
return true
|
||||
}
|
||||
if (!error) return false
|
||||
const err = error as Record<string, unknown>
|
||||
if (err.isNetworkError === true) return true
|
||||
if (err.name === 'S3NetworkError' || err.name === 'NetworkError') return true
|
||||
const msg = (typeof err.message === 'string' ? err.message : '').toLowerCase()
|
||||
if (
|
||||
msg.includes('network error') ||
|
||||
msg.includes('failed to fetch') ||
|
||||
msg.includes('networkerror') ||
|
||||
(msg.includes('load failed') && !msg.includes('upload failed')) ||
|
||||
msg.includes('err_internet_disconnected')
|
||||
) {
|
||||
|
||||
if (err.code === 'NETWORK' || err.code === 'NETWORK_ERROR') return true
|
||||
if (err.name === 'S3NetworkError' || err.name === 'UploadNetworkError') {
|
||||
return true
|
||||
}
|
||||
const req = err.request as Record<string, unknown> | undefined
|
||||
if (req && req.status === 0) return true
|
||||
|
||||
// Standard fetch network failure
|
||||
if (error instanceof TypeError) {
|
||||
return true
|
||||
}
|
||||
|
||||
// If there was no HTTP status attached, check generic Uppy NetworkError flag
|
||||
if (err.name === 'NetworkError' || err.isNetworkError === true) return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -183,21 +280,34 @@ export async function uploadFilesWithUppy({
|
||||
}
|
||||
}
|
||||
|
||||
const res = await fetch(`/api/teams/${teamId}/upload/sign`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
key: req.key,
|
||||
method: req.method,
|
||||
uploadId,
|
||||
partNumber,
|
||||
fileId,
|
||||
}),
|
||||
credentials: 'include',
|
||||
})
|
||||
let res: Response
|
||||
try {
|
||||
res = await fetch(`/api/teams/${teamId}/upload/sign`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
key: req.key,
|
||||
method: req.method,
|
||||
uploadId,
|
||||
partNumber,
|
||||
fileId,
|
||||
}),
|
||||
credentials: 'include',
|
||||
})
|
||||
} catch (err) {
|
||||
if (err instanceof TypeError) {
|
||||
throw new UploadNetworkError('Network error while signing request', { cause: err })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to sign request: ${res.statusText}`)
|
||||
const isTransient =
|
||||
res.status === 429 || res.status === 502 || res.status === 503 || res.status === 504
|
||||
throw new UploadHttpError(`Failed to sign request: ${res.statusText}`, {
|
||||
status: res.status,
|
||||
isTransient,
|
||||
})
|
||||
}
|
||||
|
||||
const data = (await res.json()) as { url: string }
|
||||
@@ -235,6 +345,7 @@ export async function uploadFilesWithUppy({
|
||||
|
||||
const confirmationPromises = new Map<string, Promise<void>>()
|
||||
const retryCounts = new Map<string, number>()
|
||||
const pendingNetworkRetries = new Set<string>()
|
||||
let retryTimeout: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const triggerRetry = () => {
|
||||
@@ -290,6 +401,10 @@ export async function uploadFilesWithUppy({
|
||||
uppy.on('upload-success', (file) => {
|
||||
if (!file) return
|
||||
const fileId = file.meta.fileId as string
|
||||
if (fileId) {
|
||||
pendingNetworkRetries.delete(fileId)
|
||||
retryCounts.delete(fileId)
|
||||
}
|
||||
activeUploads.delete(fileId)
|
||||
|
||||
const confirmPromise = (async () => {
|
||||
@@ -371,6 +486,9 @@ export async function uploadFilesWithUppy({
|
||||
console.warn(
|
||||
`[Uploader] Localhost storage endpoint mismatch detected for ${file.name}. Failing permanently without retry.`,
|
||||
)
|
||||
if (fileId) {
|
||||
pendingNetworkRetries.delete(fileId)
|
||||
}
|
||||
handlePermanentError(file, error)
|
||||
return
|
||||
}
|
||||
@@ -380,8 +498,11 @@ export async function uploadFilesWithUppy({
|
||||
`[Uploader] Network error detected for ${file.name}. Keeping upload active to resume when connection is restored.`,
|
||||
)
|
||||
|
||||
if (fileId) {
|
||||
pendingNetworkRetries.add(fileId)
|
||||
}
|
||||
|
||||
if (typeof navigator !== 'undefined' && navigator.onLine) {
|
||||
const fileId = file.meta?.fileId as string | undefined
|
||||
const attempts = (fileId ? retryCounts.get(fileId) || 0 : 0) + 1
|
||||
if (fileId) {
|
||||
retryCounts.set(fileId, attempts)
|
||||
@@ -396,6 +517,9 @@ export async function uploadFilesWithUppy({
|
||||
return
|
||||
}
|
||||
|
||||
if (fileId) {
|
||||
pendingNetworkRetries.delete(fileId)
|
||||
}
|
||||
handlePermanentError(file, error)
|
||||
})
|
||||
|
||||
@@ -443,32 +567,24 @@ export async function uploadFilesWithUppy({
|
||||
}
|
||||
|
||||
const failedFiles = result?.failed || []
|
||||
const isOffline = typeof navigator !== 'undefined' && !navigator.onLine
|
||||
const hasNetworkFailures =
|
||||
isOffline ||
|
||||
failedFiles.some((f) => {
|
||||
const err = (f as { error?: unknown })?.error
|
||||
const fileMeta = (f as { meta?: Record<string, unknown> })?.meta
|
||||
const fId = fileMeta?.fileId as string | undefined
|
||||
const uUrl =
|
||||
(fileMeta?.url as string | undefined) ||
|
||||
(fId ? activeUploads.get(fId)?.url : undefined)
|
||||
if (isLocalhostStorageMismatch(uUrl)) {
|
||||
return false
|
||||
}
|
||||
return isNetworkIssue(err)
|
||||
})
|
||||
|
||||
if (hasNetworkFailures) {
|
||||
console.log(
|
||||
'[Uploader] Batch completed with network issues; waiting for reconnection to resume...',
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
for (const f of failedFiles) {
|
||||
const file = f as { name: string; meta: Record<string, unknown> }
|
||||
handlePermanentError(file, (f as { error?: unknown })?.error)
|
||||
const fId = file.meta?.fileId as string | undefined
|
||||
if (fId && !pendingNetworkRetries.has(fId)) {
|
||||
const err = (f as { error?: unknown })?.error
|
||||
if (isNetworkIssue(err)) {
|
||||
pendingNetworkRetries.add(fId)
|
||||
continue
|
||||
}
|
||||
handlePermanentError(file, err)
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingNetworkRetries.size > 0) {
|
||||
console.log(
|
||||
`[Uploader] Batch completed with ${pendingNetworkRetries.size} retrying file(s); waiting for reconnection or backoff to resume...`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (activeUploads.size === 0) {
|
||||
|
||||
Reference in New Issue
Block a user