feat(upload): add 12-hour upload expiration and continuous backoff retry (#508)

* feat(upload): add 12-hour upload expiration and continuous backoff retry

* fix(upload): clamp jittered backoff delay to maxDelayMs and update presign JSDoc
This commit is contained in:
Yiling-J
2026-10-30 16:06:34 +08:00
committed by GitHub
parent 3b00a42a89
commit 9529c63c0a
7 changed files with 263 additions and 36 deletions
+2
View File
@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Added
- **upload**: Improve upload resilience during network drops and device sleep by adding continuous exponential backoff retries and extending upload link expiration to 12 hours
### Fixed
- **upload**: Display a clear diagnostic error toast when remote file uploads fail due to an unconfigured or localhost S3 storage endpoint
+10 -9
View File
@@ -38,15 +38,16 @@ Shumai is configured via environment variables. You can define these variables i
Shumai supports storing files locally or on S3-compatible endpoints.
| Variable | Description | Example / Default |
| :------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------ |
| `STORAGE_BACKEND` | Storage type. Use `local` for local file system storage, or `s3` for S3-compatible cloud storage. | `local` (default) or `s3` |
| `AWS_ENDPOINT_URL_S3` | Endpoint for S3 storage or local storage. Used by the browser to build file upload URLs. Must include port if deploying remotely or using custom ports. | `http://localhost:3000` (respects `SHUMAI_SERVER_PORT` fallback if unset) |
| `S3_BUCKET` | S3 bucket name. | `shumai` (default) |
| `S3_ACCESS_KEY_ID` | Access key credentials for S3. | `minioadmin` (default) |
| `S3_SECRET_ACCESS_KEY` | Secret key credentials for S3. | `minioadmin` (default) |
| `S3_REGION` | S3 storage region. | `auto` (default) |
| `PRESIGNED_URL_EXPIRES_IN` | Number of hours for which generated presigned download/upload links are valid. | `5` (default) |
| Variable | Description | Example / Default |
| :-------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------ |
| `STORAGE_BACKEND` | Storage type. Use `local` for local file system storage, or `s3` for S3-compatible cloud storage. | `local` (default) or `s3` |
| `AWS_ENDPOINT_URL_S3` | Endpoint for S3 storage or local storage. Used by the browser to build file upload URLs. Must include port if deploying remotely or using custom ports. | `http://localhost:3000` (respects `SHUMAI_SERVER_PORT` fallback if unset) |
| `S3_BUCKET` | S3 bucket name. | `shumai` (default) |
| `S3_ACCESS_KEY_ID` | Access key credentials for S3. | `minioadmin` (default) |
| `S3_SECRET_ACCESS_KEY` | Secret key credentials for S3. | `minioadmin` (default) |
| `S3_REGION` | S3 storage region. | `auto` (default) |
| `PRESIGNED_URL_EXPIRES_IN` | Number of hours for which generated presigned download links are valid. | `5` (default) |
| `PRESIGNED_UPLOAD_URL_EXPIRES_IN` | Number of hours for which generated presigned upload links are valid. Falls back to `PRESIGNED_URL_EXPIRES_IN` if unset. | `12` (default) |
### Local Storage Port Configuration
+10 -9
View File
@@ -13,15 +13,16 @@ Using external S3 storage is highly recommended for production teams. It allows
To enable S3-compatible storage, update the following environment variables in your `.env` file or Docker Compose configuration:
| Variable | Description | Required | Default / Example |
| :------------------------- | :------------------------------------------------------------------------ | :------- | :----------------------------------- |
| `STORAGE_BACKEND` | Set to `s3` to enable S3 compatible storage. | Yes | `s3` |
| `S3_BUCKET` | The name of the S3 bucket where files will be stored. | Yes | `shumai-bucket` |
| `S3_REGION` | The region of your S3 bucket. | No | `auto` |
| `S3_ACCESS_KEY_ID` | The access key ID for your storage service. | Yes | `your-access-key-id` |
| `S3_SECRET_ACCESS_KEY` | The secret access key for your storage service. | Yes | `your-secret-access-key` |
| `AWS_ENDPOINT_URL_S3` | The endpoint URL of your S3 compatible storage provider. | Yes | `https://s3.us-east-1.amazonaws.com` |
| `PRESIGNED_URL_EXPIRES_IN` | Duration in hours before generated presigned download/upload URLs expire. | No | `5` |
| Variable | Description | Required | Default / Example |
| :-------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :------- | :----------------------------------- |
| `STORAGE_BACKEND` | Set to `s3` to enable S3 compatible storage. | Yes | `s3` |
| `S3_BUCKET` | The name of the S3 bucket where files will be stored. | Yes | `shumai-bucket` |
| `S3_REGION` | The region of your S3 bucket. | No | `auto` |
| `S3_ACCESS_KEY_ID` | The access key ID for your storage service. | Yes | `your-access-key-id` |
| `S3_SECRET_ACCESS_KEY` | The secret access key for your storage service. | Yes | `your-secret-access-key` |
| `AWS_ENDPOINT_URL_S3` | The endpoint URL of your S3 compatible storage provider. | Yes | `https://s3.us-east-1.amazonaws.com` |
| `PRESIGNED_URL_EXPIRES_IN` | Duration in hours before generated presigned download URLs expire. | No | `5` |
| `PRESIGNED_UPLOAD_URL_EXPIRES_IN` | Duration in hours before generated presigned upload URLs expire. Falls back to `PRESIGNED_URL_EXPIRES_IN` if unset. | No | `12` |
---
+73 -1
View File
@@ -40,6 +40,7 @@ describe('S3Service implementations', () => {
vi.clearAllMocks()
getSignedUrlSpy.mockReset()
delete process.env.PRESIGNED_URL_EXPIRES_IN
delete process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN
getSignedUrlSpy.mockResolvedValue('http://presigned-url')
s3SendSpy.mockResolvedValue({})
})
@@ -146,7 +147,7 @@ describe('S3Service implementations', () => {
expect(getSignedUrlSpy).toHaveBeenCalledTimes(2)
})
it('should respect PRESIGNED_URL_EXPIRES_IN env', async () => {
it('should respect PRESIGNED_URL_EXPIRES_IN env for GET', async () => {
process.env.PRESIGNED_URL_EXPIRES_IN = '10'
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
@@ -159,6 +160,77 @@ describe('S3Service implementations', () => {
expect.objectContaining({ expiresIn: 10 * 3600 }),
)
delete process.env.PRESIGNED_URL_EXPIRES_IN
delete process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN
})
it('should default PUT presign and multipart to 12 hours (43200 seconds)', async () => {
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
getSignedUrlSpy.mockClear()
await s3.presign('bucket', 'key', 'PUT')
expect(getSignedUrlSpy).toHaveBeenCalledWith(
expect.anything(),
expect.any(PutObjectCommand),
expect.objectContaining({ expiresIn: 12 * 3600 }),
)
getSignedUrlSpy.mockClear()
await s3.presignMultipart('bucket', 'key', {
method: 'PUT',
key: 'key',
fileId: 'file-1',
uploadId: 'up-1',
partNumber: 1,
})
expect(getSignedUrlSpy).toHaveBeenCalledWith(
expect.anything(),
expect.any(UploadPartCommand),
expect.objectContaining({ expiresIn: 12 * 3600 }),
)
})
it('should respect PRESIGNED_UPLOAD_URL_EXPIRES_IN env for PUT and multipart', async () => {
process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN = '24'
process.env.PRESIGNED_URL_EXPIRES_IN = '5'
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
getSignedUrlSpy.mockClear()
await s3.presign('bucket', 'key', 'PUT')
expect(getSignedUrlSpy).toHaveBeenCalledWith(
expect.anything(),
expect.any(PutObjectCommand),
expect.objectContaining({ expiresIn: 24 * 3600 }),
)
getSignedUrlSpy.mockClear()
await s3.presignMultipart('bucket', 'key', {
method: 'PUT',
key: 'key',
fileId: 'file-1',
})
expect(getSignedUrlSpy).toHaveBeenCalledWith(
expect.anything(),
expect.any(PutObjectCommand),
expect.objectContaining({ expiresIn: 24 * 3600 }),
)
delete process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN
delete process.env.PRESIGNED_URL_EXPIRES_IN
})
it('should allow PRESIGNED_URL_EXPIRES_IN to override PUT if PRESIGNED_UPLOAD_URL_EXPIRES_IN is unset', async () => {
process.env.PRESIGNED_URL_EXPIRES_IN = '8'
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
getSignedUrlSpy.mockClear()
await s3.presign('bucket', 'key', 'PUT')
expect(getSignedUrlSpy).toHaveBeenCalledWith(
expect.anything(),
expect.any(PutObjectCommand),
expect.objectContaining({ expiresIn: 8 * 3600 }),
)
delete process.env.PRESIGNED_URL_EXPIRES_IN
})
+15 -4
View File
@@ -112,7 +112,7 @@ export interface S3Service {
download?: boolean,
filename?: string,
internal?: boolean,
/** URL lifetime; defaults to PRESIGNED_URL_EXPIRES_IN hours. Ignored by local storage. */
/** URL lifetime; defaults to PRESIGNED_UPLOAD_URL_EXPIRES_IN (12h) for PUT, or PRESIGNED_URL_EXPIRES_IN (5h) for GET. Ignored by local storage. */
expiresInSeconds?: number,
) => Promise<string>
presignMultipart: (
@@ -364,8 +364,16 @@ export class S3StorageService implements S3Service {
internal?: boolean,
expiresInSeconds?: number,
): Promise<string> {
const expireHours = parseInt(process.env.PRESIGNED_URL_EXPIRES_IN || '5', 10)
const lifetimeSeconds = expiresInSeconds ?? expireHours * 3600
const defaultExpireHours =
method === 'PUT'
? parseInt(
process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN ||
process.env.PRESIGNED_URL_EXPIRES_IN ||
'12',
10,
)
: parseInt(process.env.PRESIGNED_URL_EXPIRES_IN || '5', 10)
const lifetimeSeconds = expiresInSeconds ?? defaultExpireHours * 3600
// With an explicit (usually short) lifetime, only reuse a cached URL while it has at
// least half of its lifetime left, so callers never receive a nearly expired link.
const cacheTtlMs = expiresInSeconds ? (lifetimeSeconds * 1000) / 2 : lifetimeSeconds * 1000
@@ -416,7 +424,10 @@ export class S3StorageService implements S3Service {
key: string,
request: S3SignRequest,
): Promise<{ url: string }> {
const expireHours = parseInt(process.env.PRESIGNED_URL_EXPIRES_IN || '5', 10)
const expireHours = parseInt(
process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN || process.env.PRESIGNED_URL_EXPIRES_IN || '12',
10,
)
const expiresInSeconds = expireHours * 3600
let url: string
+128
View File
@@ -5,6 +5,9 @@ import {
uploadFilesWithUppy,
isNetworkIssue,
isLocalhostStorageMismatch,
calculateBackoffDelay,
MAX_RETRY_DELAY_MS,
BASE_RETRY_DELAY_MS,
} from './uploader'
import { acquireWakeLock, releaseWakeLock } from './wake-lock'
import { useUploadStore } from '@/ui/stores/upload'
@@ -272,6 +275,31 @@ describe('Uploader', () => {
})
})
describe('calculateBackoffDelay', () => {
it('increases delay with attempts and caps at MAX_RETRY_DELAY_MS (60s)', () => {
const delay1 = calculateBackoffDelay(1)
expect(delay1).toBeGreaterThanOrEqual(1700)
expect(delay1).toBeLessThanOrEqual(2300)
const delay2 = calculateBackoffDelay(2)
expect(delay2).toBeGreaterThanOrEqual(3400)
expect(delay2).toBeLessThanOrEqual(4600)
const delay3 = calculateBackoffDelay(3)
expect(delay3).toBeGreaterThanOrEqual(6800)
expect(delay3).toBeLessThanOrEqual(9200)
const delay10 = calculateBackoffDelay(10)
expect(delay10).toBeGreaterThanOrEqual(Math.round(MAX_RETRY_DELAY_MS * 0.85))
expect(delay10).toBeLessThanOrEqual(MAX_RETRY_DELAY_MS)
})
it('never returns less than baseDelayMs', () => {
const delay0 = calculateBackoffDelay(0)
expect(delay0).toBeGreaterThanOrEqual(BASE_RETRY_DELAY_MS)
})
})
it('pauses on network disconnect, does not report terminal failure, and resumes on back-online', async () => {
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
useUploadStore
@@ -462,4 +490,104 @@ describe('Uploader', () => {
addFileSpy.mockRestore()
retryAllSpy.mockRestore()
})
it('retries continuously on transient network errors while online without capping at 3 attempts', async () => {
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
useUploadStore
.getState()
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'test.mp4', size: 100 }])
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
const retryAllSpy = vi.spyOn(Uppy.prototype, 'retryAll').mockResolvedValue(undefined as never)
const uploadPromise = uploadFilesWithUppy({
files: [{ id: 'temp-1', file: new File(['content'], 'test.mp4', { type: 'video/mp4' }) }],
taskId: 'task-1',
teamId: 'team-1',
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/test.mp4' }],
})
await new Promise((r) => setTimeout(r, 10))
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
expect(capturedUppy).toBeDefined()
const mockFile = {
id: 'f1',
name: 'test.mp4',
meta: { fileId: 'file-1' },
} as unknown as UppyFile<Meta, Record<string, never>>
const networkError = new Error('Failed to fetch')
// Simulate 5 consecutive network errors while online (previously would have aborted after 3)
for (let i = 1; i <= 5; i++) {
capturedUppy.emit('upload-error', mockFile, networkError)
}
// Should still be active and uploading, NOT permanently failed
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('uploading')
expect(activeUploads.has('file-1')).toBe(true)
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).not.toHaveBeenCalled()
// Now succeed on retry
capturedUppy.emit('upload-success', mockFile, { status: 200 })
capturedUppy.emit('complete', { successful: [mockFile], failed: [] })
await uploadPromise
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('completed')
expect(activeUploads.size).toBe(0)
addFileSpy.mockRestore()
retryAllSpy.mockRestore()
})
it('halts retry loop and fails permanently when S3 returns 403 (presigned URL expired)', async () => {
Object.defineProperty(navigator, 'onLine', { value: true, configurable: true })
useUploadStore
.getState()
.startTask('task-1', 'task-1', [{ fileId: 'file-1', name: 'expired.mp4', size: 100 }])
const addFileSpy = vi.spyOn(Uppy.prototype, 'addFile').mockReturnValue('f1')
const uploadPromise = uploadFilesWithUppy({
files: [{ id: 'temp-1', file: new File(['content'], 'expired.mp4', { type: 'video/mp4' }) }],
taskId: 'task-1',
teamId: 'team-1',
storageBackend: 's3',
createdAssets: [{ tempId: 'temp-1', assetId: 'file-1', key: 'files/expired.mp4' }],
})
await new Promise((r) => setTimeout(r, 10))
const capturedUppy = addFileSpy.mock.contexts[0] as Uppy
const mockFile = {
id: 'f1',
name: 'expired.mp4',
meta: { fileId: 'file-1' },
} as unknown as UppyFile<Meta, Record<string, never>>
// S3 returns 403 when presigned URL expires after 12h
const expiredError = Object.assign(new Error('Request has expired'), { status: 403 })
capturedUppy.emit('upload-error', mockFile, expiredError)
capturedUppy.emit('complete', {
successful: [],
failed: [{ ...mockFile, error: 'Request has expired' }],
})
await uploadPromise
expect(client.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({
param: { teamId: 'team-1', taskId: 'task-1' },
json: { fileId: 'file-1', errorMessage: 'Request has expired' },
})
expect(toast.error).toHaveBeenCalledWith(
m.failed_upload_file_with_name({ name: 'expired.mp4' }),
)
expect(useUploadStore.getState().tasks['task-1']?.files['file-1']?.status).toBe('failed')
expect(activeUploads.size).toBe(0)
addFileSpy.mockRestore()
})
})
+25 -13
View File
@@ -67,6 +67,19 @@ if (typeof window !== 'undefined') {
const LOCALHOST_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]', '0.0.0.0'])
export const MAX_RETRY_DELAY_MS = 60000
export const BASE_RETRY_DELAY_MS = 1000
export function calculateBackoffDelay(
attempts: number,
maxDelayMs = MAX_RETRY_DELAY_MS,
baseDelayMs = BASE_RETRY_DELAY_MS,
): number {
const exponentialDelay = Math.min(Math.pow(2, attempts) * baseDelayMs, maxDelayMs)
const jitter = exponentialDelay * (Math.random() * 0.3 - 0.15) // ±15%
return Math.min(maxDelayMs, Math.max(baseDelayMs, Math.round(exponentialDelay + jitter)))
}
export function isLocalhostStorageMismatch(uploadUrlString?: string): boolean {
if (!uploadUrlString || typeof window === 'undefined') return false
try {
@@ -222,7 +235,6 @@ export async function uploadFilesWithUppy({
const confirmationPromises = new Map<string, Promise<void>>()
const retryCounts = new Map<string, number>()
const MAX_ONLINE_RETRIES = 3
let retryTimeout: ReturnType<typeof setTimeout> | null = null
const triggerRetry = () => {
@@ -248,6 +260,11 @@ export async function uploadFilesWithUppy({
const handleBackOnline = () => {
const isOnline = typeof navigator !== 'undefined' ? navigator.onLine : true
console.log(`[Uploader] Network 'online' event received! navigator.onLine = ${isOnline}`)
retryCounts.clear()
if (retryTimeout) {
clearTimeout(retryTimeout)
retryTimeout = null
}
triggerRetry()
}
@@ -364,22 +381,17 @@ export async function uploadFilesWithUppy({
)
if (typeof navigator !== 'undefined' && navigator.onLine) {
const fileId = file.meta.fileId as string
const attempts = (retryCounts.get(fileId) || 0) + 1
retryCounts.set(fileId, attempts)
if (attempts > MAX_ONLINE_RETRIES) {
console.error(
`[Uploader] Exceeded max online retries (${MAX_ONLINE_RETRIES}) for ${file.name}. Failing permanently.`,
)
handlePermanentError(file, error)
return
const fileId = file.meta?.fileId as string | undefined
const attempts = (fileId ? retryCounts.get(fileId) || 0 : 0) + 1
if (fileId) {
retryCounts.set(fileId, attempts)
}
const delayMs = calculateBackoffDelay(attempts)
console.log(
`[Uploader] Scheduling retry for ${file.name} (attempt ${attempts}/${MAX_ONLINE_RETRIES})...`,
`[Uploader] Scheduling retry for ${file.name} (attempt ${attempts}, delay ${delayMs}ms)...`,
)
scheduleRetry(2000)
scheduleRetry(delayMs)
}
return
}