Compare commits

...
6 Commits
Author SHA1 Message Date
yiling 0c0b887b8d Release v0.6.4 2026-11-05 22:45:25 +08:00
Yiling-J f304caea00 fix(auth,web): gracefully degrade MCP on remote HTTP and cache production static assets (#519)
* fix: gracefully handle remote HTTP URLs for MCP and cache production static assets

* test(mcp): add remote http mcp integration test and fix static etag normalization and fallback error
2026-11-05 22:21:11 +08:00
Yiling-J 65e2ac1383 fix(security): improve access control and storage security (#518)
* fix(security): improve access control and storage security

* fix(security): set NODE_ENV in dockerfiles and sign local storage get urls with lru cache
2026-11-05 20:49:38 +08:00
Yiling-J 0dc0bbeb03 feat(file-browser): add context-aware copy submenu (#517)
* feat(file-browser): add context-aware copy submenu

* refactor(file-browser): address code review feedback for copy submenu

* fix(webui): align submenu trigger text and widen file context menu

* test(e2e): guard clipboard permissions in copy-menu spec for cross-browser support
2026-11-05 09:57:34 +08:00
Yiling-J 2ecbd49d18 feat(storage): add team auto-archive assets policy and background job (#516)
* feat(storage): add team auto-archive assets policy and background job

* fix(storage): address code review feedback on lock safety, starvation, and permissions

* docs: update readme and expand archive storage documentation with auto-archive details

* feat(storage): log info message on auto-archive job start and finish with archived count

* style(storage): stack age threshold and grace period vertically with updated grace period description

* fix(storage): only log auto-archive info messages when candidate assets are found to process
2026-11-04 14:40:44 +08:00
Yiling-J ba3c6dbfee feat(storage): add archive storage support for assets (#514)
* feat(storage): add archive storage support for assets

* fix(webui): show archive icon in front of file name in breadcrumb bar

* fix(webui): display archive icon in breadcrumbs when restoring

* refactor(transcode): simplify restore asset workflow to single while loop

* docs(configuration): add archive storage documentation and readme entries

* fix(webui): show archive icon in front of version names and prevent version card overflow

* fix(webui): fix overflow in manage versions dialog scroll area

* fix(webui): prevent manage versions dialog overflow on long file names

* docs(configuration): fix archive storage environment variables and defaults

* fix(storage): address code review findings for archive storage
2026-11-03 20:31:56 +08:00
179 changed files with 12522 additions and 599 deletions
+17
View File
@@ -12,6 +12,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Changed
## [0.6.4] - 2026-11-05
### Added
- **storage**: Add optional archive storage support (`local` folder/NAS or `s3` with configurable storage class and Glacier restore) allowing editors and owners to archive and restore original files while keeping previews, proxy streams, comments, and metadata available in main storage
- **storage**: Add team-level automatic asset archival based on age policies (creation time or last modified time) with configurable post-restore grace periods, an hourly background scanning worker, and a dedicated Storage settings page displaying read-only archive backend status and policy configurations
### Fixed
- **security**: ⚠️ Several security improvements and access control hardenings. Upgrading to this release is strongly recommended.
- **mcp**: Gracefully disable MCP OAuth 2.1 with an error log when deploying on remote HTTP IP addresses without breaking the main server or API token MCP access
- **web**: Add immutable cache headers and conditional 304 Not Modified support for production JavaScript and CSS chunks to prevent unnecessary refetches on page reload
### Changed
- **file-browser**: Replace the legacy combined "Copy name and download link" action with a context-aware Copy submenu in file browser context menus and card action dropdowns, enabling users to copy image, asset names, in-app links, download links, IDs, Markdown links and storage paths(team-owner only)
## [0.6.3] - 2026-11-01
### Added
+2
View File
@@ -30,6 +30,8 @@ RUN bun run scripts/extract-runtime-deps.ts
# Stage 2: Runner
FROM oven/bun:1.4.4 AS runner
ENV NODE_ENV=production
# Install system dependencies and Jellyfin-FFmpeg (with VA-API support)
RUN apt-get update && apt-get install -y --no-install-recommends \
python3 \
+2
View File
@@ -25,6 +25,8 @@ RUN bun run scripts/extract-runtime-deps.ts
# Stage 2: Runner
FROM oven/bun:1.4.4 AS runner
ENV NODE_ENV=production
# Install transcoding system dependencies and Jellyfin-FFmpeg (with VA-API/hardware acceleration support)
RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
+2 -1
View File
@@ -26,6 +26,7 @@
Shumai is an open-source Frame.io alternative built for creative teams to collaborate and share media with ease:
* **S3-Compatible & Local Storage**: Store and serve creative assets securely using the local filesystem or any S3-compatible storage service, including AWS S3, Cloudflare R2, and MinIO.
* **Archive Storage & Auto-Archival**: Offload original master media files to secondary local storage or cold S3 storage—manually or automatically via team lifecycle policies—to save on storage costs while keeping proxies for instant browsing and playback. See [Archive Storage documentation](https://docs.shumai.one/configuration/archive-storage).
* **Frame-by-Frame Annotations & Comments**: Leave precise feedback with frame-specific drawing tools and timestamped comments on both video and image assets.
* **Secure Sharing & Collections**: Create private share links and curated media collections for clients, stakeholders, and collaborators.
* **Granular Access Control**: Manage workspace permissions with team-level and project-level role-based access control.
@@ -108,7 +109,7 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon
> By default, the Docker Compose setups use CPU software transcoding. If your server has a GPU (NVIDIA NVENC, Intel QuickSync, AMD/Intel VA-API, or Rockchip RKMPP) and you want to enable GPU hardware acceleration, check the [Hardware Transcoding Guide](https://docs.shumai.one/configuration/hardware-transcoding).
3. Configure environment variables:
* `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments.
* `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments. Note: Remote deployments must use HTTPS for MCP OAuth 2.1; if an HTTP IP address or domain is used, MCP OAuth 2.1 is gracefully disabled while the Web UI and MCP API tokens remain functional.
* `AWS_ENDPOINT_URL_S3`: By default, `AWS_ENDPOINT_URL_S3` is set to `http://localhost:{SHUMAI_SERVER_PORT}`. When you use local storage (the default storage backend with `STORAGE_BACKEND: local`) and expose Shumai on a remote server, custom domain, or custom host/port combination, set `AWS_ENDPOINT_URL_S3` to the external URL that client browsers will use to upload and download files.
For example, if you change the port mapping in `docker-compose.yaml` from `3000:3000` to `12345:3000` and deploy on a server with IP address `12.34.56.78`, set:
+2 -1
View File
@@ -26,6 +26,7 @@
Shumai 是一个专为广告,短剧,漫剧等各类创意团队打造的开源协作管理平台,功能上类似阅流和分秒帧,提供了协作、审片和分享等各种功能:
* **支持 S3 和本地存储**:你可以把创意素材安全地存放在本地硬盘或者任意兼容 S3 的存储服务里,比如 阿里云 OSS 或 MinIO 等。
* **冷归档存储**:支持将原始大文件移至冷存储(如本地挂载的 NAS、AWS S3 Glacier、Google Cloud Storage Coldline 等归档存储类型)以节省主存储成本,同时保留主存储中的预览图、代理流和元数据,不影响网页浏览与即时播放。详情请参阅[归档存储文档](https://docs.shumai.one/configuration/archive-storage)。
* **帧级别标注和评论**:支持按帧进行精确反馈,同时支持直接在视频上进行标注,让团队成员能更精确地Get到你的想法。
* **安全的分享**:可以生成私密分享链接,同时还支持给分享的图片/视频添加自定义水印,并将水印配置保存为模板以便重复使用。方便给客户、同事和合作方查看。
* **细粒度权限控制**:支持按团队、项目设置角色权限,方便管理不同成员的访问范围。
@@ -126,7 +127,7 @@ https://gh-proxy.com/https://raw.githubusercontent.com/shumaiOne/shumai/main/REA
3. 配置环境变量:
* `BETTER_AUTH_URL`:Shumai 服务的外部公开访问地址(例如 `https://shumai.example.com` 或 `http://12.34.56.78:3000`)。在远程服务器或生产环境中部署时必须配置该变量。
* `BETTER_AUTH_URL`:Shumai 服务的外部公开访问地址(例如 `https://shumai.example.com` 或 `http://12.34.56.78:3000`)。在远程服务器或生产环境中部署时必须配置该变量。注意:MCP OAuth 2.1 依赖 HTTPS(本地 loopback 除外);若远程部署使用 HTTP,系统会优雅禁用 MCP OAuth 2.1 并记录日志,Web UI 与基于 API Token 的 MCP 访问不受影响。
* `AWS_ENDPOINT_URL_S3`:默认情况下指向 `http://localhost:{SHUMAI_SERVER_PORT}`。当你使用本地存储(即默认配置 `STORAGE_BACKEND: local`)并在远程服务器、自定义域名或自定义端口上公开访问 Shumai 时,请将 `AWS_ENDPOINT_URL_S3` 设置为客户端浏览器上传和下载文件所使用的外部 URL。
例如,如果你将 `docker-compose.yaml` 中的端口映射从 `3000:3000` 修改为了 `12345:3000`,且服务部署在 IP 为 `12.34.56.78` 的服务器上,请进行如下设置:
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/agent-app",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/cli",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/premiere-uxp",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "commonjs",
"scripts": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/transcode-app",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"bin": {
+3
View File
@@ -90,6 +90,7 @@ apps/web/e2e/
| Owner removes version from 2-version stack which dissolves into standalone files | `tests/project/version-stack-manage.spec.ts` |
| Dissolving a shared version stack renders the remaining standalone file name on the share page | `tests/project/version-stack-manage.spec.ts` |
| Moving a shared file into a shared version stack leaves only 1 stack card on the share page | `tests/project/version-stack-manage.spec.ts` |
| Manage versions dialog constrains card width and truncates long file names without dialog overflow | `tests/project/version-stack-manage.spec.ts` |
| Owner searches the file list by an English name keyword | `tests/project/search-file-list.spec.ts` |
| Owner searches the file list by Chinese, Japanese, and Korean name keywords | `tests/project/search-file-list.spec.ts` |
| Owner searches the file list with a keyword and a filter condition | `tests/project/search-file-list.spec.ts` |
@@ -97,6 +98,8 @@ apps/web/e2e/
| Image share with watermark + download disabled: no download buttons on public list/detail pages, watermarked proxy served, download-url rejected | `tests/project/share-watermark-no-download.spec.ts` |
| Video share with watermark + download disabled: no download buttons on public list/detail pages, watermarked proxy served, download-url rejected | `tests/project/share-watermark-no-download.spec.ts` |
| Owner views a file and it appears in recents | `tests/project/recents.spec.ts` |
| Owner archives a file to archive storage and restores it back to main storage | `tests/project/archive-restore.spec.ts` |
| Owner opens Copy submenu in context menu and copies file name | `tests/project/copy-menu.spec.ts` |
### file
| Test | File |
+9
View File
@@ -10,12 +10,16 @@ export default async function globalTeardown() {
if (fs.existsSync(envPath)) {
const content = fs.readFileSync(envPath, 'utf8')
let bucketName: string | undefined
let archiveRoot: string | undefined
content.split('\n').forEach((line) => {
const parts = line.split('=')
if (parts.length >= 2 && parts[0].trim() === 'S3_BUCKET') {
bucketName = parts.slice(1).join('=').trim()
}
if (parts.length >= 2 && parts[0].trim() === 'ARCHIVE_LOCAL_ROOT') {
archiveRoot = parts.slice(1).join('=').trim()
}
})
if (bucketName) {
@@ -26,6 +30,11 @@ export default async function globalTeardown() {
}
}
if (archiveRoot && fs.existsSync(archiveRoot)) {
console.log(`Cleaning up E2E archive storage root: ${archiveRoot}`)
fs.rmSync(archiveRoot, { recursive: true, force: true })
}
// Delete the temp .env.e2e file
fs.unlinkSync(envPath)
}
+9 -3
View File
@@ -13,8 +13,11 @@ async function start() {
// Generate a random S3 bucket name
const randomSuffix = randomBytes(4).toString('hex')
const bucketName = `shumai-e2e-${randomSuffix}`
const archiveRoot = path.resolve('data', `shumai-e2e-archive-${randomSuffix}`)
process.env.S3_BUCKET = bucketName
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
process.env.SHUMAI_SERVER_PORT = '5200'
process.env.BETTER_AUTH_URL = 'http://localhost:5200'
process.env.NODE_ENV = 'development'
@@ -22,9 +25,12 @@ async function start() {
process.env.ENABLE_EMBEDDING_AGENT = 'false'
// Write variables to .env.e2e file so the teardown can load them
const envContent = [`DATABASE_URL=${process.env.DATABASE_URL}`, `S3_BUCKET=${bucketName}`].join(
'\n',
)
const envContent = [
`DATABASE_URL=${process.env.DATABASE_URL}`,
`S3_BUCKET=${bucketName}`,
`ARCHIVE_STORAGE_BACKEND=local`,
`ARCHIVE_LOCAL_ROOT=${archiveRoot}`,
].join('\n')
fs.writeFileSync(path.resolve(currentDir, '.env.e2e'), envContent)
console.log(`E2E env file written. Starting Web App...`)
@@ -0,0 +1,65 @@
import { expect, test } from '../../fixtures'
import { fileCard } from '../../helpers/files'
test('owner archives a file to archive storage and restores it back to main storage', async ({
file,
prisma,
}) => {
const { page, projectId, fileName, fileId } = file
await page.goto(`/projects/${projectId}`)
const card = fileCard(page, fileName)
await expect(card).toBeVisible()
// 1. Archive via context menu + confirmation dialog
await card.click({ button: 'right' })
await page.getByTestId('context-menu-archive').click()
const confirmDialog = page.getByTestId('archive-confirm-dialog')
await expect(confirmDialog).toBeVisible()
await page.getByTestId('archive-confirm-submit').click()
// Wait for the DBOS archive workflow to finish and the card badge to settle to Archived
await expect(card.getByTestId('file-card-archived-badge')).toBeVisible({ timeout: 15000 })
const archivedAsset = await prisma.asset.findUniqueOrThrow({
where: { id: fileId },
include: { storageKey: true },
})
expect(archivedAsset.status).toBe('archived')
expect(archivedAsset.storageKey?.isArchived).toBe(true)
// 2. Open detail page and verify Archived header badge + disabled original download button
await card.dblclick()
await expect(page.getByTestId('detail-header-archived-badge')).toBeVisible()
await expect(page.getByTestId('default-viewer-download-button')).toBeDisabled()
// 3. Return to project file list and restore from archive
await page.goto(`/projects/${projectId}`)
await expect(card.getByTestId('file-card-archived-badge')).toBeVisible()
await card.click({ button: 'right' })
await page.getByTestId('context-menu-restore-archive').click()
await expect(confirmDialog).toBeVisible()
await page.getByTestId('archive-confirm-submit').click()
// Wait for the DBOS restore workflow to finish and the Archived badge to disappear
await expect(card.getByTestId('file-card-archived-badge')).not.toBeVisible({ timeout: 15000 })
await expect
.poll(
async () => {
const restored = await prisma.asset.findUnique({
where: { id: fileId },
include: { storageKey: true },
})
return {
status: restored?.status,
isArchived: restored?.storageKey?.isArchived,
}
},
{ timeout: 10000 },
)
.toEqual({ status: 'processed', isArchived: false })
})
@@ -0,0 +1,43 @@
import { expect, test } from '../../fixtures'
import { fileCard } from '../../helpers/files'
test('owner sees Copy submenu in context menu and copies file name', async ({
file,
browserName,
}) => {
const { page, projectId, fileName } = file
if (browserName === 'chromium') {
await page.context().grantPermissions(['clipboard-read', 'clipboard-write'])
}
await page.goto(`/projects/${projectId}`)
await expect(fileCard(page, fileName)).toBeVisible()
// Right-click context menu
await fileCard(page, fileName).click({ button: 'right' })
const copySubTrigger = page.getByRole('menuitem', { name: /^Copy$|^复制$/i })
await expect(copySubTrigger).toBeVisible()
await copySubTrigger.click()
// Verify submenu items are visible
await expect(page.getByRole('menuitem', { name: /^Name$|^名称$/i })).toBeVisible()
await expect(page.getByRole('menuitem', { name: /^Link$|^链接$/i })).toBeVisible()
await expect(page.getByRole('menuitem', { name: /^Download link$|^下载链接$/i })).toBeVisible()
await expect(page.getByRole('menuitem', { name: /^ID$/i })).toBeVisible()
await expect(
page.getByRole('menuitem', { name: /^Markdown link$|^Markdown 链接$/i }),
).toBeVisible()
await expect(page.getByRole('menuitem', { name: /^Storage path$|^存储路径$/i })).toBeVisible()
// Click "Name"
await page.getByRole('menuitem', { name: /^Name$|^名称$/i }).click()
const toast = page.locator('[data-sonner-toast]').filter({
hasText: /Copied name to clipboard|已复制名称到剪贴板/i,
})
await expect(toast).toBeVisible()
if (browserName === 'chromium') {
const clipboardText = await page.evaluate(() => navigator.clipboard.readText())
expect(clipboardText).toBe(fileName)
}
})
@@ -231,3 +231,76 @@ test('moving a shared file into a shared version stack leaves only 1 stack card
const updatedShareRoot = await prisma.asset.findUnique({ where: { id: share.rootFolderId } })
expect(updatedShareRoot?.fileCount).toBe(1)
})
test('manage versions dialog constrains card width and truncates long file names without dialog overflow', async ({
project,
prisma,
}) => {
const { page, projectId, rootFolderId } = project
const longName =
'super-extremely-long-asset-name-for-testing-manage-versions-dialog-card-overflow-issue-that-keeps-expanding-without-truncating-properly.mp4'
const fileA = await seedFile(prisma, projectId, rootFolderId, longName)
const fileB = await seedFile(prisma, projectId, rootFolderId, `short-${Date.now()}.mp4`)
await page.goto(`/projects/${projectId}`)
await expect(fileCard(page, fileA.name)).toBeVisible()
await expect(fileCard(page, fileB.name)).toBeVisible()
// Create version stack: drag B on A
await dragCard(page, fileCard(page, fileB.name), fileCard(page, fileA.name))
await expect(page.getByText('v2', { exact: true })).toBeVisible()
const stack = await prisma.asset.findFirstOrThrow({
where: { projectId, parentId: rootFolderId, type: 'version_stack' },
})
// Navigate to file viewer for the stack
await page.goto(`/projects/${projectId}/files/${stack.id}`)
// Open breadcrumb menu and click "Manage versions..."
await page.getByRole('button', { name: new RegExp(fileB.name) }).click()
await page.getByRole('menuitem', { name: 'Versions', exact: true }).hover()
await page.getByRole('button', { name: /manage versions$/i }).click()
const dialog = page.getByRole('dialog')
await expect(dialog).toBeVisible()
const rowA = dialog.locator('div.group', { hasText: longName })
await expect(rowA).toBeVisible()
// Measure the dialog and version row
const metrics = await page.evaluate(() => {
const dialogEl = document.querySelector('[role="dialog"]') as HTMLElement | null
if (!dialogEl) return null
const row = dialogEl.querySelector('div.group') as HTMLElement | null
return {
dialogScrollWidth: dialogEl.scrollWidth,
dialogClientWidth: dialogEl.clientWidth,
dialogOffsetWidth: dialogEl.offsetWidth,
rowScrollWidth: row?.scrollWidth,
rowClientWidth: row?.clientWidth,
}
})
expect(metrics).not.toBeNull()
// Dialog should not horizontally scroll or overflow
expect(metrics!.dialogScrollWidth).toBeLessThanOrEqual(metrics!.dialogClientWidth)
// Row clientWidth should be bounded within the dialog
expect(metrics!.rowClientWidth).toBeLessThanOrEqual(metrics!.dialogClientWidth)
expect(metrics!.rowScrollWidth).toBeLessThanOrEqual(metrics!.rowClientWidth!)
// Hover over the row and verify the more-options button is visible and within bounds
await rowA.hover()
const moreButton = rowA.getByRole('button', { name: 'More options' })
await expect(moreButton).toBeVisible()
const moreBtnBox = await moreButton.boundingBox()
const dialogBox = await dialog.boundingBox()
expect(moreBtnBox).not.toBeNull()
expect(dialogBox).not.toBeNull()
expect(moreBtnBox!.x + moreBtnBox!.width).toBeLessThanOrEqual(dialogBox!.x + dialogBox!.width)
// Click more options button and verify menu opens
await moreButton.click()
await expect(page.getByRole('menuitem', { name: /remove from stack/i })).toBeVisible()
})
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/web-app",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"dependencies": {
+30 -15
View File
@@ -8,6 +8,7 @@ import index from '@shumai/webui/index.html'
import { initAgentWorkflows } from '@shumai/agent'
import { app } from '@shumai/api'
import { assetService } from '@shumai/core/src/asset/asset'
import { validateArchiveStorageConfig } from '@shumai/core/src/s3/archive-storage'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { initTranscodeWorkflows } from '@shumai/transcode'
import { workflowService } from '@shumai/workflow-core'
@@ -19,6 +20,8 @@ import { authService } from '@shumai/core/src/auth/auth'
import { embeddingService } from '@shumai/core/src/search/embedding-service'
import { sandboxService } from '@shumai/core'
import { notificationJobService } from '@shumai/core/src/notification/notification-job'
import { autoArchiveJobService } from '@shumai/core/src/asset/auto-archive-job'
import { createHtmlHandler, createStaticFileHandler } from './static'
if (process.argv.includes('--check')) {
console.log('✅ Web app evaluated successfully!')
@@ -76,6 +79,7 @@ if (regenCmdIndex !== -1) {
}
async function run() {
validateArchiveStorageConfig()
// Initialize workflows and activities for local executor mode
initAgentWorkflows()
initTranscodeWorkflows()
@@ -89,6 +93,7 @@ async function run() {
await agentService.ensureSystemEmbeddingAgents().catch(console.error)
assetService.startCleanupJob()
notificationJobService.start()
autoArchiveJobService.start()
const args = process.argv.slice(2)
let workersOption = process.env.SHUMAI_WORKERS || 'all'
for (let i = 0; i < args.length; i++) {
@@ -99,13 +104,15 @@ async function run() {
}
}
let listenQueues: string[] = ['agent_queue', 'transcode_queue']
let listenQueues: string[] = ['agent_queue', 'transcode_queue', 'archive_queue']
if (workersOption === 'none') {
listenQueues = []
} else if (workersOption === 'agent' || workersOption === 'ai') {
listenQueues = ['agent_queue']
listenQueues = ['agent_queue', 'archive_queue']
} else if (workersOption === 'transcode') {
listenQueues = ['transcode_queue']
} else if (workersOption === 'archive') {
listenQueues = ['archive_queue']
}
workflowService.start(listenQueues)
@@ -145,7 +152,8 @@ async function run() {
for (const file of bundle.files) {
const routePath = file.path.startsWith('.') ? file.path.slice(1) : file.path
const filePath = join(import.meta.dir, file.path)
routes[routePath] = () => new Response(Bun.file(filePath), { headers: file.headers })
const isHtml = file.loader === 'html' || file.path.endsWith('.html')
routes[routePath] = createStaticFileHandler(filePath, file.headers, !isHtml)
}
const sampleFile = bundle.files[0]?.path || bundle.index || ''
@@ -157,13 +165,14 @@ async function run() {
const routePath = '/' + pdfWorkerFile
const filePath = join(bundleDir, pdfWorkerFile)
const bunFile = Bun.file(filePath)
routes[routePath] = () =>
new Response(bunFile, {
headers: {
'content-type': 'text/javascript;charset=utf-8',
etag: `"${bunFile.size}-${bunFile.lastModified}"`,
},
})
routes[routePath] = createStaticFileHandler(
filePath,
{
'content-type': 'text/javascript;charset=utf-8',
etag: `"${bunFile.size}-${bunFile.lastModified}"`,
},
true,
)
}
} catch (err) {
console.warn('Failed to register pdf.worker route:', err)
@@ -179,16 +188,22 @@ async function run() {
.replaceAll('href="./', 'href="/')
.replaceAll('src="./', 'src="/')
.replaceAll('content="./', 'content="/')
const serveMainHtml = () => new Response(mainHtmlText, { headers: mainHtmlHeaders })
const serveMainHtml = createHtmlHandler(mainHtmlText, mainHtmlHeaders)
routes['/'] = serveMainHtml
routes['/*'] = serveMainHtml
} else {
const fallbackHtmlPath = join(import.meta.dir, 'shumai-app.html')
const serveFallbackHtml = () =>
new Response(Bun.file(fallbackHtmlPath), {
headers: { 'content-type': 'text/html;charset=utf-8' },
})
const fallbackHtmlFile = Bun.file(fallbackHtmlPath)
if (!(await fallbackHtmlFile.exists())) {
throw new Error(
`Production HTML bundle is missing: neither bundle.files nor ${fallbackHtmlPath} exists.`,
)
}
const fallbackRaw = await fallbackHtmlFile.text()
const serveFallbackHtml = createHtmlHandler(fallbackRaw, {
'content-type': 'text/html;charset=utf-8',
})
routes['/'] = serveFallbackHtml
routes['/*'] = serveFallbackHtml
}
+175
View File
@@ -0,0 +1,175 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { writeFileSync, unlinkSync } from 'node:fs'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { createHtmlHandler, createStaticFileHandler, isEtagMatching, normalizeEtag } from './static'
describe('Static Asset Serving & Caching', () => {
const tempChunkPath = join(tmpdir(), `test-chunk-${Date.now()}.js`)
const chunkContent = 'console.log("chunk loaded");'
beforeAll(() => {
writeFileSync(tempChunkPath, chunkContent, 'utf8')
})
afterAll(() => {
try {
unlinkSync(tempChunkPath)
} catch {
// Ignore cleanup error if already deleted
}
})
describe('normalizeEtag', () => {
it('quotes unquoted ETags according to RFC 9110', () => {
expect(normalizeEtag('hHRmXFeFVSk')).toBe('"hHRmXFeFVSk"')
expect(normalizeEtag('ACJrxcts9Iw')).toBe('"ACJrxcts9Iw"')
})
it('preserves already-quoted strong and weak ETags', () => {
expect(normalizeEtag('"hHRmXFeFVSk"')).toBe('"hHRmXFeFVSk"')
expect(normalizeEtag('W/"hHRmXFeFVSk"')).toBe('W/"hHRmXFeFVSk"')
})
it('returns undefined for empty or missing ETags', () => {
expect(normalizeEtag(undefined)).toBeUndefined()
expect(normalizeEtag('')).toBeUndefined()
expect(normalizeEtag(' ')).toBeUndefined()
})
})
describe('isEtagMatching', () => {
it('matches exact etag string', () => {
expect(isEtagMatching('"12345"', '"12345"')).toBe(true)
})
it('matches weak etag to strong or weak candidates', () => {
expect(isEtagMatching('W/"12345"', '"12345"')).toBe(true)
expect(isEtagMatching('"12345"', 'W/"12345"')).toBe(true)
expect(isEtagMatching('W/"12345"', 'W/"12345"')).toBe(true)
})
it('matches wildcard *', () => {
expect(isEtagMatching('"12345"', '*')).toBe(true)
})
it('matches comma-separated candidate lists', () => {
expect(isEtagMatching('"12345"', '"old-etag", "12345", "other"')).toBe(true)
expect(isEtagMatching('"12345"', '"old-etag", "mismatch"')).toBe(false)
})
it('returns false when etag or ifNoneMatch is missing or empty', () => {
expect(isEtagMatching(undefined, '"12345"')).toBe(false)
expect(isEtagMatching('"12345"', null)).toBe(false)
expect(isEtagMatching('', '"12345"')).toBe(false)
expect(isEtagMatching('"12345"', '')).toBe(false)
})
})
describe('createStaticFileHandler', () => {
it('sets immutable Cache-Control for immutable content-hashed assets', async () => {
const handler = createStaticFileHandler(
tempChunkPath,
{ 'content-type': 'text/javascript;charset=utf-8', etag: '"chunk-hash-1"' },
true,
)
const req = new Request('http://localhost/chunk-123.js')
const res = handler(req)
expect(res.status).toBe(200)
expect(res.headers.get('content-type')).toBe('text/javascript;charset=utf-8')
expect(res.headers.get('cache-control')).toBe('public, max-age=31536000, immutable')
expect(res.headers.get('etag')).toBe('"chunk-hash-1"')
expect(await res.text()).toBe(chunkContent)
})
it('returns 304 Not Modified when If-None-Match matches the file etag', async () => {
const handler = createStaticFileHandler(
tempChunkPath,
{ 'content-type': 'text/javascript;charset=utf-8', etag: '"chunk-hash-1"' },
true,
)
const req = new Request('http://localhost/chunk-123.js', {
headers: { 'if-none-match': '"chunk-hash-1"' },
})
const res = handler(req)
expect(res.status).toBe(304)
expect(res.headers.get('cache-control')).toBe('public, max-age=31536000, immutable')
expect(res.headers.get('etag')).toBe('"chunk-hash-1"')
expect(await res.text()).toBe('')
})
it('sets no-cache Cache-Control for non-immutable static assets', async () => {
const handler = createStaticFileHandler(
tempChunkPath,
{ 'content-type': 'text/plain', etag: '"plain-1"' },
false,
)
const req = new Request('http://localhost/robots.txt')
const res = handler(req)
expect(res.status).toBe(200)
expect(res.headers.get('cache-control')).toBe('no-cache')
})
it('normalizes unquoted manifest ETags to quoted RFC 9110 strings', async () => {
const handler = createStaticFileHandler(
tempChunkPath,
{ 'content-type': 'text/javascript;charset=utf-8', etag: 'hHRmXFeFVSk' },
true,
)
const req = new Request('http://localhost/chunk-123.js')
const res = handler(req)
expect(res.status).toBe(200)
expect(res.headers.get('etag')).toBe('"hHRmXFeFVSk"')
const revalidateRes = handler(
new Request('http://localhost/chunk-123.js', {
headers: { 'if-none-match': '"hHRmXFeFVSk"' },
}),
)
expect(revalidateRes.status).toBe(304)
expect(revalidateRes.headers.get('etag')).toBe('"hHRmXFeFVSk"')
})
})
describe('createHtmlHandler', () => {
const htmlText = '<!DOCTYPE html><html><head><title>Shumai</title></head><body></body></html>'
it('sets no-cache and generates deterministic ETag for HTML', async () => {
const handler = createHtmlHandler(htmlText, { 'content-type': 'text/html;charset=utf-8' })
const req = new Request('http://localhost/')
const res = handler(req)
expect(res.status).toBe(200)
expect(res.headers.get('content-type')).toBe('text/html;charset=utf-8')
expect(res.headers.get('cache-control')).toBe('no-cache')
const etag = res.headers.get('etag')
expect(etag).toBeTruthy()
expect(await res.text()).toBe(htmlText)
})
it('returns 304 Not Modified when HTML If-None-Match matches', async () => {
const handler = createHtmlHandler(htmlText, { 'content-type': 'text/html;charset=utf-8' })
const initialRes = handler(new Request('http://localhost/'))
const etag = initialRes.headers.get('etag')!
const revalidateReq = new Request('http://localhost/', {
headers: { 'if-none-match': etag },
})
const revalidateRes = handler(revalidateReq)
expect(revalidateRes.status).toBe(304)
expect(revalidateRes.headers.get('cache-control')).toBe('no-cache')
expect(await revalidateRes.text()).toBe('')
})
})
})
+106
View File
@@ -0,0 +1,106 @@
/**
* Helpers for serving static production assets with caching headers and ETag revalidation.
*/
/**
* Checks whether an incoming `If-None-Match` header matches the asset's ETag.
* Supports weak ETags (`W/"..."`), exact matches, and comma-separated lists.
*/
export function isEtagMatching(etag: string | undefined, ifNoneMatch: string | null): boolean {
if (!etag || !ifNoneMatch) {
return false
}
const cleanTarget = etag.trim().replace(/^W\//, '').replace(/^"|"$/g, '')
if (!cleanTarget) {
return false
}
const candidates = ifNoneMatch
.split(',')
.map((candidate) => candidate.trim().replace(/^W\//, '').replace(/^"|"$/g, ''))
return candidates.includes(cleanTarget) || candidates.includes('*')
}
/**
* Normalizes an ETag to ensure it is enclosed in quotes according to RFC 9110.
* Preserves weak validators (`W/"..."`).
*/
export function normalizeEtag(etag: string | undefined): string | undefined {
if (!etag) {
return undefined
}
const trimmed = etag.trim()
if (!trimmed) {
return undefined
}
if (trimmed.startsWith('W/"') && trimmed.endsWith('"')) {
return trimmed
}
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
return trimmed
}
return `"${trimmed.replace(/^"|"$/g, '')}"`
}
/**
* Creates a route handler for a static file on disk.
* Content-hashed assets (e.g. `chunk-*.js`, `chunk-*.css`) should be marked `isImmutable: true`
* to be aggressively cached locally by browsers across page reloads.
*/
export function createStaticFileHandler(
filePath: string,
baseHeaders: Record<string, string>,
isImmutable: boolean,
): (req: Request) => Response {
const bunFile = Bun.file(filePath)
const rawEtag = baseHeaders.etag || baseHeaders.ETag
const normalizedEtag = normalizeEtag(rawEtag)
const headers: Record<string, string> = {
...baseHeaders,
...(normalizedEtag ? { etag: normalizedEtag } : {}),
'cache-control': isImmutable ? 'public, max-age=31536000, immutable' : 'no-cache',
}
const etag = headers.etag
return (req: Request): Response => {
const ifNoneMatch = req.headers.get('if-none-match')
if (etag && isEtagMatching(etag, ifNoneMatch)) {
return new Response(null, {
status: 304,
headers,
})
}
return new Response(bunFile, { headers })
}
}
/**
* Creates a route handler for in-memory transformed HTML entry points.
* Sets `cache-control: no-cache` with a deterministic ETag, allowing browsers to cache locally
* while always revalidating to ensure new chunk hashes are picked up immediately after deployment.
*/
export function createHtmlHandler(
htmlText: string,
baseHeaders: Record<string, string>,
): (req: Request) => Response {
const rawEtag = baseHeaders.etag || baseHeaders.ETag
const etag = normalizeEtag(rawEtag) || `"${Bun.hash(htmlText).toString(16)}"`
const headers: Record<string, string> = {
...baseHeaders,
etag,
'cache-control': 'no-cache',
}
return (req: Request): Response => {
const ifNoneMatch = req.headers.get('if-none-match')
if (isEtagMatching(etag, ifNoneMatch)) {
return new Response(null, {
status: 304,
headers,
})
}
return new Response(htmlText, { headers })
}
}
+16 -16
View File
@@ -32,7 +32,7 @@
},
"apps/agent": {
"name": "@shumai/agent-app",
"version": "0.6.3",
"version": "0.6.4",
"bin": {
"shumai-agent": "./src/index.ts",
},
@@ -44,7 +44,7 @@
},
"apps/cli": {
"name": "@shumai/cli",
"version": "0.6.3",
"version": "0.6.4",
"bin": {
"shumai-cli": "./src/index.ts",
},
@@ -61,7 +61,7 @@
},
"apps/premiere-uxp": {
"name": "@shumai/premiere-uxp",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@shumai/api": "workspace:*",
"@shumai/dtos": "workspace:*",
@@ -115,7 +115,7 @@
},
"apps/transcode": {
"name": "@shumai/transcode-app",
"version": "0.6.3",
"version": "0.6.4",
"bin": {
"shumai-transcode": "./src/index.ts",
},
@@ -127,7 +127,7 @@
},
"apps/web": {
"name": "@shumai/web-app",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@shumai/agent": "workspace:*",
"@shumai/api": "workspace:*",
@@ -147,7 +147,7 @@
},
"packages/agent": {
"name": "@shumai/agent",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@anthropic-ai/sandbox-runtime": "0.0.74",
"@earendil-works/pi-agent-core": "0.82.1",
@@ -171,7 +171,7 @@
},
"packages/api": {
"name": "@shumai/api",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@better-auth/mcp": "1.7.12",
"@hono/zod-validator": "0.7.6",
@@ -191,7 +191,7 @@
},
"packages/core": {
"name": "@shumai/core",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@ai-sdk/alibaba": "2.0.38",
"@ai-sdk/amazon-bedrock": "5.0.67",
@@ -255,7 +255,7 @@
},
"packages/db": {
"name": "@shumai/db",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"@prisma/adapter-pg": "7.10.0",
@@ -271,7 +271,7 @@
},
"packages/dtos": {
"name": "@shumai/dtos",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"zod": "4.5.4",
@@ -279,7 +279,7 @@
},
"packages/e2e": {
"name": "@shumai/e2e",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"@google/genai": "2.28.0",
@@ -294,18 +294,18 @@
},
"packages/operations": {
"name": "@shumai/operations",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"zod": "4.5.4",
},
},
"packages/tableprinter": {
"name": "@shumai/tableprinter",
"version": "0.6.3",
"version": "0.6.4",
},
"packages/transcode": {
"name": "@shumai/transcode",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -316,7 +316,7 @@
},
"packages/webui": {
"name": "@shumai/webui",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@base-ui/react": "1.7.0",
"@better-auth/oauth-provider": "1.7.12",
@@ -419,7 +419,7 @@
},
"packages/workflow-core": {
"name": "@shumai/workflow-core",
"version": "0.6.3",
"version": "0.6.4",
"dependencies": {
"@dbos-inc/dbos-sdk": "5.2.11",
"@shumai/core": "workspace:*",
+19
View File
@@ -3,6 +3,25 @@ title: "Changelog"
description: "Recent updates, new features, and improvements in Shumai"
---
<Update label="v0.6.4" description="November 5, 2026">
### Added
- **storage**: Add optional archive storage support (`local` folder/NAS or `s3` with configurable storage class and Glacier restore) allowing editors and owners to archive and restore original files while keeping previews, proxy streams, comments, and metadata available in main storage
- **storage**: Add team-level automatic asset archival based on age policies (creation time or last modified time) with configurable post-restore grace periods, an hourly background scanning worker, and a dedicated Storage settings page displaying read-only archive backend status and policy configurations
### Fixed
- **security**: ⚠️ Several security improvements and access control hardenings. Upgrading to this release is strongly recommended.
- **mcp**: Gracefully disable MCP OAuth 2.1 with an error log when deploying on remote HTTP IP addresses without breaking the main server or API token MCP access
- **web**: Add immutable cache headers and conditional 304 Not Modified support for production JavaScript and CSS chunks to prevent unnecessary refetches on page reload
### Changed
- **file-browser**: Replace the legacy combined "Copy name and download link" action with a context-aware Copy submenu in file browser context menus and card action dropdowns, enabling users to copy image, asset names, in-app links, download links, IDs, Markdown links and storage paths(team-owner only)
</Update>
<Update label="v0.6.3" description="November 1, 2026">
### Added
+250
View File
@@ -0,0 +1,250 @@
---
title: Archive Storage
description: Configure secondary cold storage to offload raw media files while preserving in-browser proxy playback and previews.
---
High-resolution camera footage, RAW image sequences, master audio stems, and large 3D assets quickly consume primary storage. Shumai's **Archive Storage** feature enables creative teams to offload heavy master original files to cost-effective cold storage (such as a local NAS, AWS S3 Glacier, or Google Cloud Storage Coldline) while keeping all proxy media, preview images, waveform files, and metadata in primary storage.
Team members can continue to browse, search, scrub, and review archived assets in the web browser without interruption. When the original raw file is needed for mastering or color grading, it can be restored back to primary storage with a single click.
---
## How It Works
1. **Original-Only Archiving**: Only the original raw file is moved to archive storage. Previews, thumbnails, and transcoded proxies (e.g., 180p scrubbing proxies, 1080p streamable MP4s, WebP posters) remain in primary storage.
2. **Durable Asynchronous Workflows**: Moving and restoring files is handled by background worker tasks on a dedicated queue (`archive_queue`) managed by DBOS. Tasks automatically survive network interruptions, slow S3 transfers, and server restarts.
3. **Transparent Asset Lifecycle**:
- When an asset is being archived, its status transitions to `archiving` and settles at `archived`.
- When an asset is being restored, its status transitions to `restoring` and returns to `processed` once the original file is back in primary storage.
- Trashed assets that are permanently purged automatically delete their original file from archive storage as well.
4. **Folder & Version Stack Support**: Archiving a folder or version stack recursively archives all contained file assets. Restoring a folder or version stack restores all archived files inside it.
---
## Storage Topologies
Shumai supports flexible storage configurations where primary storage and archive storage can use different backends or storage providers:
| Topology | Main Storage | Archive Storage | Description |
| :------------------------------------ | :----------- | :-------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
| **Local NAS / Directory** | Local or S3 | `local` | Offload files to a mounted network folder (NFS / SMB) or a secondary cold disk array. |
| **Separate S3 Bucket / Provider** | S3 or Local | `s3` | Stream files to a dedicated archive bucket on AWS S3, Wasabi, Backblaze B2, or Google Cloud Storage. |
| **Same S3 Bucket (In-Place Tiering)** | AWS S3 | `s3` | Re-tier objects in-place on AWS S3 using cold storage classes (`GLACIER`, `GLACIER_IR`, `DEEP_ARCHIVE`) without re-uploading bytes over the network. |
---
## Configuration Variables
Configure archive storage by adding the following environment variables to your `.env` file or Docker Compose configuration.
### General Configuration
| Variable | Description | Required | Default |
| :------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :--------------- |
| `ARCHIVE_STORAGE_BACKEND` | Storage backend for archives: `local` or `s3`. When unset, archive functionality is disabled in the WebUI. | No | Unset (Disabled) |
| `CONCURRENCY_ARCHIVE` | Maximum concurrent archive and restore tasks executed by background workers. | No | `5` |
| `SHUMAI_WORKERS` | Worker queues listened to by the application server: `all` (default), `none`, `agent` (or `ai`), `transcode`, `archive`. Set to `all` or `archive` to process tasks. | No | `all` |
| `AUTO_ARCHIVE_INTERVAL_MINUTES` | Frequency (in minutes) of the background auto-archive scanner job. Accepts decimal numbers (e.g. `0.02` for testing). | No | `60` (1 hour) |
### Local Archive Storage
Used when `ARCHIVE_STORAGE_BACKEND=local`:
| Variable | Description | Required | Default |
| :------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :--------------- |
| `ARCHIVE_LOCAL_ROOT` | Host filesystem directory path where archived files are stored. Must not overlap with main local storage directory (`./data` or `./data/<S3_BUCKET>`). | Yes | Unset (Required) |
### S3 Archive Storage
Used when `ARCHIVE_STORAGE_BACKEND=s3`. Note that archive storage maintains its own configuration and **never falls back** to the primary `S3_*` environment variables:
| Variable | Description | Required | Default |
| :--------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :------------------------------------ |
| `ARCHIVE_S3_BUCKET` | Destination S3 bucket for archived files (can be a dedicated bucket or the same bucket as `S3_BUCKET` for in-place tiering). | Yes | Unset (Required) |
| `ARCHIVE_S3_ACCESS_KEY_ID` | Access key ID for the archive S3 service. | Yes | Unset (Required) |
| `ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret access key for the archive S3 service. | Yes | Unset (Required) |
| `ARCHIVE_AWS_ENDPOINT_URL_S3` | S3 endpoint URL for archive storage (e.g. MinIO, Cloudflare R2, Google Cloud Storage). | No | Unset (AWS standard endpoint) |
| `ARCHIVE_S3_REGION` | S3 region for the archive bucket. | No | `auto` |
| `ARCHIVE_S3_FORCE_PATH_STYLE` | Use path-style S3 URLs (`true` for MinIO/path-style, `false` for virtual-hosted style). | No | `true` |
| `ARCHIVE_S3_STORAGE_CLASS` | S3 storage class to apply to archived objects (e.g. `GLACIER`, `GLACIER_IR`, `DEEP_ARCHIVE`, `COLDLINE`). Required and must not be `STANDARD` when archiving to the same S3 bucket. | No | Unset |
| `ARCHIVE_S3_USE_RESTORE_OBJECT` | Controls whether S3 `RestoreObject` is called before reading archived objects (`auto`, `true`, or `false`). | No | `auto` |
| `ARCHIVE_S3_RESTORE_DAYS` | Number of days a restored temporary copy remains readable in S3 for asynchronous Glacier restores. | No | `1` |
| `ARCHIVE_S3_RESTORE_TIER` | S3 Glacier retrieval tier (`Expedited`, `Standard`, or `Bulk`). Omit when using providers that do not require asynchronous retrieval. | No | Unset |
| `ARCHIVE_RESTORE_POLL_INTERVAL_MS` | Milliseconds between status checks while waiting for an asynchronous cold storage restore. | No | `60000` (1 min initial, 5 min steady) |
---
## Configuration Examples
### 1. Local NAS / Mounted Path
Ideal for self-hosted teams with on-premises Network Attached Storage (Synology, QNAP, TrueNAS, or an NFS/SMB share mounted to the host machine):
```env
# Main storage on local disk or S3
STORAGE_BACKEND=local
LOCAL_STORAGE_PATH=./storage
# Archive storage pointing to your mounted NAS directory
ARCHIVE_STORAGE_BACKEND=local
ARCHIVE_LOCAL_ROOT=/mnt/nas/shumai-archive
```
In `docker-compose.yaml`, mount the host NAS volume into the container:
```yaml
services:
shumai:
volumes:
- /mnt/nas/shumai-archive:/mnt/nas/shumai-archive
environment:
ARCHIVE_STORAGE_BACKEND: local
ARCHIVE_LOCAL_ROOT: /mnt/nas/shumai-archive
```
---
### 2. AWS S3 Glacier Instant Retrieval (Archive)
AWS S3 Glacier Instant Retrieval (`GLACIER_IR`) offers cold storage pricing with **millisecond retrieval latency**. Because objects in `GLACIER_IR` do not require an asynchronous `RestoreObject` request, restoring an asset to main storage is instantaneous.
#### Same-Bucket In-Place Tiering (Fast Path)
If your main storage is AWS S3, you can tier files into `GLACIER_IR` within the **same bucket** without re-uploading bytes over the network:
```env
# Main Storage
STORAGE_BACKEND=s3
S3_BUCKET=my-shumai-production
S3_REGION=us-east-1
S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
# Archive Storage (same bucket & credentials with cold storage class)
ARCHIVE_STORAGE_BACKEND=s3
ARCHIVE_S3_BUCKET=my-shumai-production
ARCHIVE_S3_REGION=us-east-1
ARCHIVE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
ARCHIVE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
ARCHIVE_AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
ARCHIVE_S3_STORAGE_CLASS=GLACIER_IR
```
#### Dedicated Archive Bucket
If you prefer keeping archived media in a separate S3 bucket:
```env
# Archive Storage in a separate bucket
ARCHIVE_STORAGE_BACKEND=s3
ARCHIVE_S3_BUCKET=my-shumai-archive-glacier-ir
ARCHIVE_S3_REGION=us-east-1
ARCHIVE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
ARCHIVE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
ARCHIVE_AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
ARCHIVE_S3_STORAGE_CLASS=GLACIER_IR
```
---
### 3. AWS S3 Glacier Flexible Archive (Cold Storage)
For maximum cost savings on rarely accessed footage, AWS S3 Flexible Archive (`GLACIER`) or Deep Archive (`DEEP_ARCHIVE`) stores data at the lowest storage rates.
When a user restores an asset from Glacier, Shumai initiates a `RestoreObject` request, puts the background workflow into durable DBOS sleep, and periodically checks until AWS completes the retrieval:
```env
ARCHIVE_STORAGE_BACKEND=s3
ARCHIVE_S3_BUCKET=my-shumai-cold-archive
ARCHIVE_S3_REGION=us-east-1
ARCHIVE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
ARCHIVE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
ARCHIVE_AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
ARCHIVE_S3_STORAGE_CLASS=GLACIER
# Keep the restored temporary copy accessible for 7 days
ARCHIVE_S3_RESTORE_DAYS=7
# Glacier retrieval tier: Standard (3-5 hours), Expedited (1-5 min), or Bulk (5-12 hours)
ARCHIVE_S3_RESTORE_TIER=Standard
```
> [!NOTE]
> During a Glacier restore, DBOS schedules durable sleep timers in PostgreSQL. The worker thread is completely freed and does not consume concurrency slots while waiting for AWS to thaw the file.
---
### 4. Google Cloud Storage Coldline (Archive)
Google Cloud Storage (GCS) provides S3-compatible XML APIs and low-cost storage classes such as `COLDLINE` and `ARCHIVE`. Like Glacier Instant Retrieval, GCS Coldline provides **immediate access** without requiring an asynchronous thaw step:
```env
# Main storage (e.g. Cloudflare R2, MinIO, or standard GCS)
STORAGE_BACKEND=s3
S3_BUCKET=shumai-main
...
# Archive storage on GCS Coldline
ARCHIVE_STORAGE_BACKEND=s3
ARCHIVE_S3_BUCKET=shumai-coldline-archive
ARCHIVE_S3_ACCESS_KEY_ID=GOOG1EXAMPLETOCKENKEY
ARCHIVE_S3_SECRET_ACCESS_KEY=your-gcs-hmac-secret-access-key
ARCHIVE_AWS_ENDPOINT_URL_S3=https://storage.googleapis.com
ARCHIVE_S3_STORAGE_CLASS=COLDLINE
```
> [!TIP]
> Google Cloud Storage objects are readable immediately and do not accept AWS Glacier retrieval parameters. Do not set `ARCHIVE_S3_RESTORE_TIER`. If using GCS's `ARCHIVE` storage class, set `ARCHIVE_S3_USE_RESTORE_OBJECT=false` to bypass S3 `RestoreObject` calls.
---
## Automatic Asset Archival
In addition to manual on-demand archival, Shumai allows teams to automate cold storage tiering based on asset age lifecycle policies. This ensures heavy original master files are automatically offloaded to secondary cold storage without requiring manual housekeeping from creative staff or administrators.
### WebUI Configuration
Team owners can configure automatic archival under **Team Settings > Storage**:
1. **Archive Storage Status Overview**:
- Displays the current system archive storage backend (`local` or `s3`), endpoint, bucket, or root directory configured on the server.
- If archive storage is not configured in server environment variables, a warning alert is displayed, and auto-archive controls are disabled.
- For security, credentials and private server settings are not exposed to client browsers.
2. **Automatic Asset Archival Policy**:
- **Enable Auto Archive**: Toggle automated background archival on or off for the team (disabled by default; requires an active archive storage backend).
- **Archive Criteria (`method`)**:
- **Not modified for specified days (`not_updated`)**: Evaluates the last modification timestamp (`updatedAt`). Ideal for active production workflows—files stay in fast primary storage as long as team members are actively commenting, annotating, or modifying them. Once no modifications have occurred for the configured duration, the asset is moved to cold storage.
- **After creation for specified days (`after_created`)**: Evaluates the initial upload/creation timestamp (`createdAt`). Ideal for strict retention or compliance pipelines where footage is tiered based on its age (e.g., 60 or 90 days after delivery) regardless of recent review comments.
- **Age Threshold (`days`)**: The threshold in days (between 1 and 3,650 days) after which eligible assets are queued for archival.
- **Post-Restore Grace Period (`postRestoreGraceDays`)**:
- When an asset is restored from archive back to primary storage, its modification date is refreshed to the restore time.
- When using the **After creation** rule, an asset older than the threshold (e.g. 100 days old) would otherwise be immediately picked up and re-archived by the very next hourly scan.
- The post-restore grace period (default: 30 days) guarantees that restored files remain in primary storage for at least the specified number of days before becoming eligible for automated archival again. This value should be set smaller than the Age Threshold. Under this policy, only files that are older than the Age Threshold and have also surpassed the Post-Restore Grace Period since restoration are auto-archived.
### Sibling Deduplication & Safety
Shumai uses content-addressable storage keys (`storage_keys`) to deduplicate identical files across projects and folders:
- **Sibling Protection**: If multiple asset records reference the same underlying storage object (e.g., copied or duplicated files across folders or projects), the auto-archive scanner inspects all sibling assets.
- **Strict Consensus**: The underlying storage file is **only** archived if **all active sibling files** satisfy the team's auto-archive criteria and are in a stable `processed` state.
- **Starvation-Free Scheduling**: If any sibling copy was modified recently or is currently in an active workflow, the storage key is safely skipped without blocking other eligible assets from being archived.
### Background Scanner & Execution Semantics
The auto-archive job runs periodically in the background:
- **Scan Frequency**: Configured via the `AUTO_ARCHIVE_INTERVAL_MINUTES` environment variable (defaults to `60` minutes; accepts decimal values such as `0.02` for automated testing).
- **Distributed Leader Election**: Uses PostgreSQL transaction-scoped advisory locks (`pg_try_advisory_xact_lock`) during the candidate discovery phase. In multi-pod or multi-replica container deployments, only one server instance performs the scan per interval while other replicas cleanly skip.
- **Race Condition Prevention**: During archival, candidate assets transition atomically via conditional SQL (`UPDATE assets SET status = 'archiving' WHERE status = 'processed'`) protected by row-level `storage_keys FOR UPDATE` locks. This guarantees zero race conditions with concurrent manual user archival, restoration, or file deletion.
---
## WebUI Behavior & Permissions
- **Permissions**: Only team **Owners** and **Editors** can archive or restore assets.
- **Context Menus**: Right-click any file, folder, or version stack in the file browser to select **Archive** or **Restore from Archive**.
- **Visual Badges**: Archived items display an `Archived` badge on grid cards and list rows. Inside the file viewer, an archive icon is displayed in front of the file name in the breadcrumb bar.
- **Playback & Previews**: Video streaming, scrubbers, waveform audio, PDF rendering, and image viewing remain fully functional using cached proxies.
- **Download Restrictions**: The download button for the original raw file is disabled with a tooltip explaining that the raw file is archived. Transcoded proxy downloads (e.g. 1080p MP4) remain available.
+21 -1
View File
@@ -61,6 +61,25 @@ For example, if you deploy Shumai on a remote server at `http://123.456.7.8` and
- Set `SHUMAI_SERVER_PORT` to `3000`.
- Set `AWS_ENDPOINT_URL_S3` to `http://123.456.7.8:12345`.
### Archive Storage Config
Shumai supports configuring an optional **Archive Storage** tier (`local` NAS/directory or `s3` bucket/storage class) for moving original raw files while keeping proxies, previews, thumbnails, and metadata in main storage.
| Variable | Description | Example / Default |
| :------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------- |
| `ARCHIVE_STORAGE_BACKEND` | Archive storage type. Leave unset to disable archive storage, or set to `local` or `s3`. | Unset (disabled), `local`, or `s3` |
| `ARCHIVE_LOCAL_ROOT` | Required when `ARCHIVE_STORAGE_BACKEND=local`. Root directory (e.g., a mounted NAS folder) where archived original files are stored directly under `<ARCHIVE_LOCAL_ROOT>/<key>`. | `/mnt/nas/shumai-archive` |
| `ARCHIVE_AWS_ENDPOINT_URL_S3` | Optional custom S3 endpoint for archive storage when `ARCHIVE_STORAGE_BACKEND=s3` (independent of `AWS_ENDPOINT_URL_S3`). | `https://s3.us-east-1.amazonaws.com` |
| `ARCHIVE_S3_BUCKET` | Required when `ARCHIVE_STORAGE_BACKEND=s3`. Target S3 bucket for archived files (can be a dedicated bucket or the same bucket as `S3_BUCKET` with a cold storage class). | `shumai-archive` |
| `ARCHIVE_S3_ACCESS_KEY_ID` | Required when `ARCHIVE_STORAGE_BACKEND=s3`. S3 access key ID for archive storage. | `AKIA...` |
| `ARCHIVE_S3_SECRET_ACCESS_KEY` | Required when `ARCHIVE_STORAGE_BACKEND=s3`. S3 secret access key for archive storage. | `...` |
| `ARCHIVE_S3_REGION` | S3 region for archive storage when `ARCHIVE_STORAGE_BACKEND=s3`. | `auto` (default) |
| `ARCHIVE_S3_STORAGE_CLASS` | Optional S3 `StorageClass` header applied when archiving objects (accepts any standard or provider-specific class string; required and must not be `STANDARD` for same-bucket tiering). | `GLACIER_IR`, `GLACIER`, `DEEP_ARCHIVE`, `STANDARD_IA` |
| `ARCHIVE_S3_USE_RESTORE_OBJECT` | Controls whether S3 `RestoreObject` is called before reading archived objects (`auto`, `true`, or `false`). | `auto` (default) |
| `ARCHIVE_S3_RESTORE_DAYS` | Number of days a restored copy of a cold S3 object remains readable during restoration. | `1` (default) |
| `ARCHIVE_S3_RESTORE_TIER` | Optional S3 Glacier retrieval tier (`Expedited`, `Standard`, `Bulk`). Omitted when unset for S3-compatible providers that reject `GlacierJobParameters`. | Unset (default) |
| `AUTO_ARCHIVE_INTERVAL_MINUTES` | Frequency (in minutes) of the background auto-archive scanner job. Accepts decimal values (e.g. `0.02` for testing). | `60` (1 hour default) |
---
## Workflow Config
@@ -69,8 +88,9 @@ Configures background worker execution, concurrency limits, and worker identity
| Variable | Description | Example / Default |
| :---------------------- | :-------------------------------------------------------------------------------------------------------------------------------- | :---------------- |
| `SHUMAI_WORKERS` | Queues listened to by the web server. Options: `all` (default), `none` (API/UI only), `agent` (or `ai`), `transcode`. | `all` (default) |
| `SHUMAI_WORKERS` | Queues listened to by the web server. Options: `all` (default), `none` (API/UI only), `agent` (or `ai`), `transcode`, `archive`. | `all` (default) |
| `CONCURRENCY_TRANSCODE` | Maximum concurrent media transcoding tasks executed by workers. | `1` (default) |
| `CONCURRENCY_ARCHIVE` | Maximum concurrent archive and restore tasks executed by workers. | `5` (default) |
| `CONCURRENCY_AGENT` | Maximum concurrent AI tasks executed by workers. Set to `1` when running local `embedding-server`. | `5` (default) |
| `SHUMAI_WORKER_ID` | Optional explicit instance ID for worker replicas in multi-node deployments (auto-derived from hostname and process ID if unset). | `worker-1` |
| `POD_NAME` | Kubernetes pod name, used to provide stable per-pod worker identity across StatefulSet restarts. | `transcode-pod-0` |
+5 -1
View File
@@ -77,7 +77,11 @@
},
{
"group": "Configuration",
"pages": ["configuration/env-variables", "configuration/hardware-transcoding"]
"pages": [
"configuration/env-variables",
"configuration/hardware-transcoding",
"configuration/archive-storage"
]
},
{
"group": "Distributed Deployment",
+1 -1
View File
@@ -44,7 +44,7 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon
3. **Configure Environment Variables**:
Open `docker-compose.yaml` to adjust environment variables for your deployment:
- `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments.
- `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments. Note: Remote deployments must use HTTPS for MCP OAuth 2.1; if an HTTP IP address or domain is used, MCP OAuth 2.1 is gracefully disabled while the Web UI and MCP API tokens remain functional.
- `AWS_ENDPOINT_URL_S3`: By default, `AWS_ENDPOINT_URL_S3` is set to `http://localhost:{SHUMAI_SERVER_PORT}`. When you use local storage (the default storage backend with `STORAGE_BACKEND: local`) and expose Shumai on a remote server, custom domain, or custom host/port combination, set `AWS_ENDPOINT_URL_S3` to the external URL that client browsers will use to upload and download files.
For example, if you change the port mapping in `docker-compose.yaml` from `3000:3000` to `12345:3000` and deploy on a server with IP address `12.34.56.78`, set:
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "shumai",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"repository": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/agent",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -5,6 +5,7 @@ import type { AgentToolResult } from '@earendil-works/pi-agent-core'
import { downloadAsset, MAX_DOWNLOAD_ASSETS, type DownloadAssetOutput } from '@shumai/operations'
import type { User } from '@shumai/db'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { assertStorageKeyNotArchived } from '@shumai/core/src/asset/asset-archive'
import { storageBucket } from '@shumai/core/src/asset-read'
import { s3Service } from '@shumai/core/src/s3/s3'
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
@@ -54,6 +55,7 @@ async function downloadKey(key: string, userId: string): Promise<AgentToolResult
type: ResourceType.Asset,
id: owningAssetId,
})
await assertStorageKeyNotArchived(key)
const filename = sanitizeFilename(path.basename(key)) || 'downloaded_file'
const saved = await saveToSandbox(key, filename)
return {
@@ -67,6 +67,7 @@ describe('operation-backed agent tools', () => {
.map((t) => t.name)
.sort(),
).toEqual([
'archive_assets',
'copy_assets',
'create_comment',
'create_file',
@@ -87,6 +88,7 @@ describe('operation-backed agent tools', () => {
'remove_version',
'rename_asset',
'resolve_comment',
'restore_archived_assets',
'search_assets',
'update_asset_metadata',
])
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/api",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1
View File
@@ -28,6 +28,7 @@ describe('authn api', () => {
expect(await res.json()).toEqual({
initialized: true,
demoMode: false,
archiveStorageEnabled: false,
})
})
})
+17 -1
View File
@@ -2,15 +2,19 @@ import { Hono } from 'hono'
import { teamService } from '@shumai/core/src/team/team'
import { userService } from '@shumai/core/src/user/user'
import { auth } from '@shumai/core/src/auth/auth'
import { createGuestToken } from '@shumai/core/src/auth/guest-token'
import { isArchiveStorageEnabled } from '@shumai/core/src/s3/archive-storage'
import { z } from 'zod'
import { zValidator } from '@hono/zod-validator'
import { ulid } from 'ulid'
import { setCookie } from 'hono/cookie'
const app = new Hono()
const identifySchema = z.object({
username: z.string().min(1),
email: z.string().email(),
shareId: z.string().min(1).optional(),
})
const route = app
@@ -19,6 +23,7 @@ const route = app
return c.json({
initialized: info.initialized,
demoMode: info.demoMode,
archiveStorageEnabled: isArchiveStorageEnabled(),
})
})
.get('/me', async (c) => {
@@ -28,7 +33,7 @@ const route = app
return c.json({ id: session?.user?.id ?? null })
})
.post('/identify', zValidator('json', identifySchema), async (c) => {
const { username, email } = c.req.valid('json')
const { username, email, shareId } = c.req.valid('json')
const dummyEmail = `guest_${ulid()}@guest.local`
const user = await userService.createGuestUser({
@@ -37,6 +42,17 @@ const route = app
guestEmail: email,
})
if (shareId) {
const token = createGuestToken({ userId: user.id, shareId })
setCookie(c, `shumai_guest_${shareId}`, token, {
httpOnly: true,
path: '/',
sameSite: 'Lax',
secure: c.req.url.startsWith('https://') || process.env.NODE_ENV === 'production',
maxAge: 30 * 24 * 60 * 60,
})
}
return c.json({ id: user.id })
})
.all('/auth/*', (c) => {
+191 -4
View File
@@ -3,6 +3,7 @@ import { Hono, type Context, type Next } from 'hono'
import { HTTPException } from 'hono/http-exception'
import fileRoute from './file'
import { assetService } from '@shumai/core/src/asset/asset'
import { projectService } from '@shumai/core/src/project/project'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { notificationService } from '@shumai/core/src/notification/notification'
import { s3Service } from '@shumai/core/src/s3/s3'
@@ -11,8 +12,10 @@ import { authMiddleware } from './middleware/auth'
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
import type { AssetInfo } from '@shumai/dtos'
import {
archiveAssetsOp,
renameAssetOp,
resolveCommentOp,
restoreArchivedAssetsOp,
runOperation,
updateAssetMetadataOp,
} from '@shumai/core/src/operations'
@@ -817,10 +820,13 @@ describe('file api', () => {
.spyOn(assetService, 'getProjectIds')
.mockResolvedValue(['project-id-1'])
const mockGetDownloadLinks = vi.spyOn(assetService, 'getDownloadLinks').mockResolvedValue([
{ id: 'file1', name: 'file1.txt', url: 'http://link1' },
{ id: 'file2', name: 'file2.txt', url: 'http://link2' },
])
const mockGetDownloadLinks = vi.spyOn(assetService, 'getDownloadLinks').mockResolvedValue({
files: [
{ id: 'file1', name: 'file1.txt', url: 'http://link1' },
{ id: 'file2', name: 'file2.txt', url: 'http://link2' },
],
skippedArchivedCount: 0,
})
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/download-links', {
@@ -866,6 +872,127 @@ describe('file api', () => {
mockGetProjectIds.mockRestore()
})
describe('POST /files/storage-paths', () => {
it('returns empty paths if ids are empty', async () => {
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/storage-paths', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: [] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json).toEqual({ paths: [] })
})
it('rejects if assets belong to multiple projects', async () => {
const mockGetProjectIds = vi
.spyOn(assetService, 'getProjectIds')
.mockResolvedValue(['proj-1', 'proj-2'])
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/storage-paths', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: ['asset-1', 'asset-2'] }),
})
expect(res.status).toBe(400)
const json = await res.json()
expect(json.error).toBe('All selected items must belong to the same project')
mockGetProjectIds.mockRestore()
})
it('denies access if user is not a team owner (admin)', async () => {
const mockGetProjectIds = vi
.spyOn(assetService, 'getProjectIds')
.mockResolvedValue(['proj-1'])
const mockGetProjectTeam = vi
.spyOn(projectService, 'getProjectTeam')
.mockResolvedValue('team-1')
vi.mocked(authzService.hasPermission).mockRejectedValueOnce(
new HTTPException(403, { message: 'Permission denied' }),
)
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/storage-paths', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: ['asset-1'] }),
})
expect(res.status).toBe(403)
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user1', name: 'Test User' },
permission: Permission.Admin,
type: ResourceType.Team,
id: 'team-1',
})
mockGetProjectIds.mockRestore()
mockGetProjectTeam.mockRestore()
})
it('returns storage paths if user has team admin permission', async () => {
const mockGetProjectIds = vi
.spyOn(assetService, 'getProjectIds')
.mockResolvedValue(['proj-1'])
const mockGetProjectTeam = vi
.spyOn(projectService, 'getProjectTeam')
.mockResolvedValue('team-1')
const mockGetStoragePaths = vi.spyOn(assetService, 'getStoragePaths').mockResolvedValue({
paths: [
{
id: 'asset-1',
name: 'clip.mp4',
path: 's3://shumai/files/1/clip.mp4',
backend: 's3',
isArchived: false,
},
],
})
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/storage-paths', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: ['asset-1'] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json.paths).toHaveLength(1)
expect(json.paths[0].name).toBe('clip.mp4')
expect(json.paths[0].path).toBe('s3://shumai/files/1/clip.mp4')
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user1', name: 'Test User' },
permission: Permission.Admin,
type: ResourceType.Team,
id: 'team-1',
})
expect(assetService.getStoragePaths).toHaveBeenCalledWith(['asset-1'])
mockGetProjectIds.mockRestore()
mockGetProjectTeam.mockRestore()
mockGetStoragePaths.mockRestore()
})
it('rejects if more than 100 ids are provided', async () => {
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const ids = Array.from({ length: 101 }, (_, i) => `asset-${i}`)
const res = await app.request('/files/storage-paths', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids }),
})
expect(res.status).toBe(400)
})
})
describe('GET /files/:fileId/m3u8 routes', () => {
it('returns master playlist with correct content-type', async () => {
const mockMaster = vi
@@ -908,4 +1035,64 @@ describe('file api', () => {
mockVariant.mockRestore()
})
})
describe('POST /files/archive and POST /files/restore-archive routes', () => {
const validUlid = '01ARZ3NDEKTSV4RRFFQ69G5FAV'
it('archives assets via archiveAssetsOp', async () => {
vi.mocked(runOperation).mockResolvedValueOnce({
archivedAssetIds: [validUlid],
taskIds: ['task-arch-1'],
skippedCount: 0,
})
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/archive', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: [validUlid] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json).toEqual({
archivedAssetIds: [validUlid],
taskIds: ['task-arch-1'],
skippedCount: 0,
})
expect(runOperation).toHaveBeenCalledWith(
archiveAssetsOp,
{ userId: 'user1', via: 'api' },
{ assetIds: [validUlid] },
)
})
it('restores archived assets via restoreArchivedAssetsOp', async () => {
vi.mocked(runOperation).mockResolvedValueOnce({
restoredAssetIds: [validUlid],
taskIds: ['task-rest-1'],
skippedCount: 0,
})
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
const res = await app.request('/files/restore-archive', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ids: [validUlid] }),
})
expect(res.status).toBe(200)
const json = await res.json()
expect(json).toEqual({
restoredAssetIds: [validUlid],
taskIds: ['task-rest-1'],
skippedCount: 0,
})
expect(runOperation).toHaveBeenCalledWith(
restoreArchivedAssetsOp,
{ userId: 'user1', via: 'api' },
{ assetIds: [validUlid] },
)
})
})
})
+64 -2
View File
@@ -1,27 +1,33 @@
import { zValidator } from '@hono/zod-validator'
import { assetService } from '@shumai/core/src/asset/asset'
import { projectService } from '@shumai/core/src/project/project'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { commentService } from '@shumai/core/src/comment/comment'
import { s3Service } from '@shumai/core/src/s3/s3'
import type { Prisma } from '@shumai/db'
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
import {
archiveAssetsOp,
renameAssetOp,
resolveCommentOp,
restoreArchivedAssetsOp,
runOperation,
updateAssetMetadataOp,
} from '@shumai/core/src/operations'
import {
archiveAssetsRequestSchema,
createCommentRequestSchema,
deleteFilesRequestSchema,
batchGetFilesRequestSchema,
paginationParamsSchema,
restoreArchivedAssetsRequestSchema,
restoreFilesRequestSchema,
updateAssetMetadataRequestSchema,
updateAssetOrderRequestSchema,
updateFileRequestSchema,
uploadFileRequestSchema,
getDownloadLinksRequestSchema,
getStoragePathsRequestSchema,
completeCommentRequestSchema,
exportCommentsRequestSchema,
addCommentReactionRequestSchema,
@@ -343,6 +349,28 @@ const route = new Hono<{ Variables: { user: User } }>()
await assetService.restoreAssets(req.ids)
return c.body(null, 204)
})
.post('/files/archive', zValidator('json', archiveAssetsRequestSchema), async (c) => {
const req = c.req.valid('json')
const result = await runOperation(
archiveAssetsOp,
{ userId: c.get('user').id, via: 'api' },
{ assetIds: req.ids },
)
return c.json(result)
})
.post(
'/files/restore-archive',
zValidator('json', restoreArchivedAssetsRequestSchema),
async (c) => {
const req = c.req.valid('json')
const result = await runOperation(
restoreArchivedAssetsOp,
{ userId: c.get('user').id, via: 'api' },
{ assetIds: req.ids },
)
return c.json(result)
},
)
.post('/teams/:teamId/files', zValidator('form', uploadFileRequestSchema), async (c) => {
const teamId = c.req.param('teamId')
const user = c.get('user')
@@ -424,8 +452,42 @@ const route = new Hono<{ Variables: { user: User } }>()
})
// 4. Generate download links
const files = await assetService.getDownloadLinks(req.ids)
return c.json({ files })
const { files, skippedArchivedCount } = await assetService.getDownloadLinks(req.ids)
return c.json({
files,
...(skippedArchivedCount > 0 ? { skippedArchivedCount } : {}),
})
})
.post('/files/storage-paths', zValidator('json', getStoragePathsRequestSchema), async (c) => {
const user = c.get('user')
const req = c.req.valid('json')
if (req.ids.length === 0) {
return c.json({ paths: [] })
}
const projectIds = await assetService.getProjectIds(req.ids)
if (projectIds.length === 0) {
return c.json({ paths: [] })
}
if (projectIds.length !== 1) {
return c.json({ error: 'All selected items must belong to the same project' }, 400)
}
const projectId = projectIds[0]
const teamId = await projectService.getProjectTeam(projectId)
// Verify user has Admin (owner) permission on the team
await authzService.hasPermission({
user,
permission: Permission.Admin,
type: ResourceType.Team,
id: teamId,
})
const result = await assetService.getStoragePaths(req.ids)
return c.json(result)
})
.post(
'/files/download-url',
@@ -0,0 +1,100 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { Hono } from 'hono'
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'
import { prisma, type User } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { apiTokenService } from '@shumai/core/src/user/api-token'
import { projectService } from '@shumai/core/src/project/project'
describe('MCP server on remote HTTP BETTER_AUTH_URL', () => {
setupTestDbHooks()
const originalEnv = process.env.BETTER_AUTH_URL
let user: User
let token: string
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let app: Hono<any>
let isMcpOauthEnabled: boolean
beforeAll(async () => {
process.env.BETTER_AUTH_URL = 'http://43.133.100.178:3000'
vi.resetModules()
const authModule = await import('@shumai/core/src/auth/auth')
isMcpOauthEnabled = authModule.isMcpOauthEnabled
const mcpServerModule = await import('./mcp-server')
app = new Hono().route('/', mcpServerModule.default)
user = await prisma.user.create({
data: { name: 'Remote HTTP User', email: `remote-http-${Date.now()}@test.com` },
})
const team = await prisma.team.create({ data: { name: 'Remote HTTP Team' } })
await prisma.teamMember.create({ data: { teamId: team.id, userId: user.id, role: 'owner' } })
await projectService.createProject(user, { teamId: team.id, name: 'Remote HTTP Project' })
const tokenRes = await apiTokenService.createToken(user.id, 'remote-http-test')
token = tokenRes.token
})
afterAll(() => {
if (originalEnv !== undefined) {
process.env.BETTER_AUTH_URL = originalEnv
} else {
delete process.env.BETTER_AUTH_URL
}
vi.resetModules()
})
it('omits MCP OAuth plugin and disables isMcpOauthEnabled', () => {
expect(isMcpOauthEnabled).toBe(false)
})
it('returns 503 Service Unavailable for unauthenticated requests to /mcp', async () => {
const res = await app.request('/mcp', { method: 'POST' })
expect(res.status).toBe(503)
const body = await res.json()
expect(body).toMatchObject({
jsonrpc: '2.0',
error: {
code: -32000,
message: expect.stringContaining(
'MCP OAuth 2.1 is disabled on this server because BETTER_AUTH_URL is not HTTPS',
),
},
id: null,
})
})
it('allows valid API tokens to access /mcp and execute tools even when OAuth is disabled', async () => {
const transport = new StreamableHTTPClientTransport(new URL('http://localhost/mcp'), {
fetch: async (input, init) => {
const headers = new Headers(init?.headers)
headers.set('Authorization', `Bearer ${token}`)
return app.request(String(input), {
...init,
headers,
})
},
})
const client = new Client({ name: 'remote-http-client', version: '1.0.0' })
await client.connect(transport)
expect(client.getServerVersion()).toMatchObject({ name: 'shumai' })
const { tools } = await client.listTools()
expect(tools.length).toBeGreaterThan(0)
expect(tools.map((t) => t.name)).toContain('list_projects')
await client.close()
})
it('rejects invalid API tokens with 401 Unauthorized', async () => {
const res = await app.request('/mcp', {
method: 'POST',
headers: {
Authorization: 'Bearer shumai_invalidtoken1234567890',
},
})
expect(res.status).toBe(401)
})
})
+21 -1
View File
@@ -6,7 +6,7 @@ import { teamService } from '@shumai/core/src/team/team'
import { s3Service } from '@shumai/core/src/s3/s3'
import { createFolderOp, readAssetOp, runOperation } from '@shumai/core/src/operations'
import { MAX_INLINE_IMAGE_BYTES, MAX_INLINE_IMAGES } from './mcp-extensions/read-asset'
import mcpServerRoute, { MCP_SERVER_INSTRUCTIONS } from './mcp-server'
import mcpServerRoute, { disabledOauthHandler, MCP_SERVER_INSTRUCTIONS } from './mcp-server'
vi.mock('@shumai/core/src/operations', async (importOriginal) => ({
...(await importOriginal<typeof import('@shumai/core/src/operations')>()),
@@ -53,6 +53,7 @@ describe('MCP server', () => {
const { tools } = await client.listTools()
expect(tools.map((t) => t.name).sort()).toEqual([
'archive_assets',
'complete_upload',
'copy_assets',
'create_comment',
@@ -74,6 +75,7 @@ describe('MCP server', () => {
'remove_version',
'rename_asset',
'resolve_comment',
'restore_archived_assets',
'search_assets',
'update_asset_metadata',
])
@@ -400,4 +402,22 @@ describe('MCP server', () => {
])
})
})
describe('Disabled OAuth Fallback', () => {
it('returns 503 Service Unavailable with standard JSON-RPC error payload', async () => {
const res = await disabledOauthHandler()
expect(res.status).toBe(503)
expect(res.headers.get('content-type')).toContain('application/json')
const body = await res.json()
expect(body).toEqual({
jsonrpc: '2.0',
error: {
code: -32000,
message:
'MCP OAuth 2.1 is disabled on this server because BETTER_AUTH_URL is not HTTPS or a loopback address. Please configure HTTPS or authenticate with a Shumai API token.',
},
id: null,
})
})
})
})
+54 -34
View File
@@ -5,7 +5,7 @@ import type { z } from 'zod'
import type { Prisma } from '@shumai/db'
import { dropNullOptionals } from '@shumai/operations'
import { getOperationsForSurface, runOperation, type Operation } from '@shumai/core/src/operations'
import { auth, authBaseUrl, mcpResource } from '@shumai/core/src/auth/auth'
import { auth, authBaseUrl, isMcpOauthEnabled, mcpResource } from '@shumai/core/src/auth/auth'
import { apiTokenService } from '@shumai/core/src/user/api-token'
import { mcpExtensions, type McpToolExtension } from './mcp-extensions'
import { assertWritableInDemoMode, describeOperationError } from './operation-support'
@@ -134,40 +134,60 @@ const mcpHandler = createMcpHandler(({ authInfo }) => {
return buildMcpServer(userId)
})
const oauthHandler = requireMcpAuth(
auth,
(req, claims) => {
const userId = claims.sub
if (!userId) {
throw new Error('OAuth token has no sub claim')
}
const clientId = (claims.azp as string) ?? (claims.client_id as string) ?? ''
const scopes =
typeof claims.scope === 'string'
? claims.scope.split(' ')
: ((claims.scopes as string[]) ?? [])
return mcpHandler.fetch(req, {
authInfo: {
token: '',
clientId,
scopes,
extra: { userId },
export const disabledOauthHandler = async (): Promise<Response> => {
return new Response(
JSON.stringify({
jsonrpc: '2.0',
error: {
code: -32000,
message:
'MCP OAuth 2.1 is disabled on this server because BETTER_AUTH_URL is not HTTPS or a loopback address. Please configure HTTPS or authenticate with a Shumai API token.',
},
})
},
{
resource: mcpResource,
// In-process verification: avoids HTTP loopback round-trip and works even in test environments
// where no HTTP port is bound.
jwksUrl: (async () => {
const res = await auth.handler(new Request(`${authBaseUrl}/api/auth/jwks`))
if (!res.ok) {
throw new Error(`Failed to load JWKS in-process: ${res.statusText}`)
}
return await res.json()
}) as unknown as string,
},
)
id: null,
}),
{
status: 503,
headers: { 'Content-Type': 'application/json' },
},
)
}
const oauthHandler = isMcpOauthEnabled
? requireMcpAuth(
auth,
(req, claims) => {
const userId = claims.sub
if (!userId) {
throw new Error('OAuth token has no sub claim')
}
const clientId = (claims.azp as string) ?? (claims.client_id as string) ?? ''
const scopes =
typeof claims.scope === 'string'
? claims.scope.split(' ')
: ((claims.scopes as string[]) ?? [])
return mcpHandler.fetch(req, {
authInfo: {
token: '',
clientId,
scopes,
extra: { userId },
},
})
},
{
resource: mcpResource,
// In-process verification: avoids HTTP loopback round-trip and works even in test environments
// where no HTTP port is bound.
jwksUrl: (async () => {
const res = await auth.handler(new Request(`${authBaseUrl}/api/auth/jwks`))
if (!res.ok) {
throw new Error(`Failed to load JWKS in-process: ${res.statusText}`)
}
return await res.json()
}) as unknown as string,
},
)
: disabledOauthHandler
/** Shumai's MCP server (Streamable HTTP). Mounted at `/mcp` with dual API token and OAuth 2.1 auth. */
const route = new Hono<{ Variables: { user: User } }>().all('/mcp', async (c) => {
+49 -17
View File
@@ -1,5 +1,6 @@
import { Hono } from 'hono'
import type { Context } from 'hono'
import { getCookie } from 'hono/cookie'
import { zValidator } from '@hono/zod-validator'
import { shareService } from '@shumai/core/src/share/share'
import { assetService } from '@shumai/core/src/asset/asset'
@@ -12,6 +13,7 @@ import type { Prisma as PrismaType } from '@shumai/db'
import { projectService } from '@shumai/core/src/project/project'
import { userService } from '@shumai/core/src/user/user'
import { auth } from '@shumai/core/src/auth/auth'
import { verifyGuestToken } from '@shumai/core/src/auth/guest-token'
import { createCommentRequestSchema } from '@shumai/dtos'
import { notificationService } from '@shumai/core/src/notification/notification'
import {
@@ -160,7 +162,7 @@ const route = app
const password = c.req.header('x-share-password')
try {
const shareLink = await shareService.getShareLink(shareId)
await shareService.verifyPublicAccess(shareLink.rootFolderId, password)
await shareService.verifyPublicAccess(shareId, shareLink.rootFolderId, password)
return c.json({
id: shareLink.id,
name: shareLink.name,
@@ -184,7 +186,7 @@ const route = app
try {
const shareLink = await shareService.getShareLink(shareId)
await shareService.verifyPublicAccess(shareLink.rootFolderId, password)
await shareService.verifyPublicAccess(shareId, shareLink.rootFolderId, password)
const fields = await metadataService.listProjectFields(null, shareLink.projectId)
const fieldVisibility = (shareLink.fieldVisibility || {}) as Record<string, boolean>
@@ -202,13 +204,14 @@ const route = app
'/shares/:shareId/folders/:folderId/children',
zValidator('query', sharedChildrenRequestSchema),
async (c) => {
const shareId = c.req.param('shareId')
const folderId = c.req.param('folderId')
const password =
c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
const req = c.req.valid('query')
try {
const shareLink = await shareService.verifyPublicAccess(folderId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, folderId, password)
let sort: string | undefined
let order: string | undefined
@@ -264,11 +267,12 @@ const route = app
},
)
.get('/shares/:shareId/files/:fileId', async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
try {
const shareLink = await shareService.verifyPublicAccess(fileId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
let asset = await assetService.getAsset({ assetId: fileId })
@@ -303,12 +307,13 @@ const route = app
'/shares/:shareId/files/:fileId/download-url',
zValidator('json', z.object({ key: z.string().min(1) })),
async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const password = c.req.header('x-share-password')
const { key } = c.req.valid('json')
try {
const shareLink = await shareService.verifyPublicAccess(fileId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
if (!shareLink.allowDownload) {
throw new ShareLinkDownloadDisabledError('Download is disabled for this share link')
@@ -329,12 +334,13 @@ const route = app
'/shares/:shareId/files/:fileId/comments',
zValidator('query', paginationParamsSchema),
async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const password = c.req.header('x-share-password')
const req = c.req.valid('query')
try {
await shareService.verifyPublicAccess(fileId, password)
await shareService.verifyPublicAccess(shareId, fileId, password)
const targetFileId = await assetService.resolveTargetAssetId(fileId)
let userId: string | undefined
@@ -343,6 +349,18 @@ const route = app
})
if (session?.user) {
userId = session.user.id
} else {
const cookieName = `shumai_guest_${shareId}`
const token = getCookie(c, cookieName) || getCookie(c, 'shumai_guest')
if (token) {
const verified = verifyGuestToken(token, shareId)
if (verified) {
const guestUser = await userService.getUserById(verified.userId)
if (guestUser && guestUser.guestEmail) {
userId = guestUser.id
}
}
}
}
const res = await assetService.listComments(targetFileId, req, userId)
return c.json(res)
@@ -358,12 +376,15 @@ const route = app
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const password = c.req.header('x-share-password')
const guestUserId = c.req.header('x-guest-user-id')
const req = c.req.valid('json')
if (req.attachmentIds && req.attachmentIds.length > 0) {
return c.json({ error: 'Attachments are not allowed on public share comments' }, 400)
}
try {
const shareLink = await shareService.getShareLink(shareId)
await shareService.verifyPublicAccess(fileId, password)
await shareService.verifyPublicAccess(shareId, fileId, password)
const targetFileId = await assetService.resolveTargetAssetId(fileId)
@@ -377,10 +398,17 @@ const route = app
if (session?.user) {
userId = session.user.id
} else if (guestUserId) {
const guestUser = await userService.getUserById(guestUserId)
if (guestUser) {
userId = guestUser.id
} else {
const cookieName = `shumai_guest_${shareId}`
const token = getCookie(c, cookieName) || getCookie(c, 'shumai_guest')
if (token) {
const verified = verifyGuestToken(token, shareId)
if (verified) {
const guestUser = await userService.getUserById(verified.userId)
if (guestUser && guestUser.guestEmail) {
userId = guestUser.id
}
}
}
}
@@ -400,7 +428,7 @@ const route = app
const notifType = req.replyToId ? 'reply_created' : 'comment_created'
notificationService.create({
await notificationService.create({
type: notifType,
teamId: teamId,
creatorId: userId,
@@ -415,11 +443,12 @@ const route = app
},
)
.get('/shares/:shareId/files/:fileId/m3u8', async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
try {
const shareLink = await shareService.verifyPublicAccess(fileId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
let playlist = await assetService.getHlsMasterPlaylist({
assetId: targetAssetId,
@@ -438,11 +467,12 @@ const route = app
}
})
.get('/shares/:shareId/files/:fileId/m3u8/master.m3u8', async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
try {
const shareLink = await shareService.verifyPublicAccess(fileId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
let playlist = await assetService.getHlsMasterPlaylist({
assetId: targetAssetId,
@@ -461,6 +491,7 @@ const route = app
}
})
.get('/shares/:shareId/files/:fileId/m3u8/:resolution', async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
let resolution = c.req.param('resolution')
if (resolution.endsWith('.m3u8')) {
@@ -469,7 +500,7 @@ const route = app
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
try {
const shareLink = await shareService.verifyPublicAccess(fileId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
const playlist = await assetService.getHlsVariantPlaylist({
assetId: targetAssetId,
@@ -483,12 +514,13 @@ const route = app
}
})
.get('/shares/:shareId/files/:fileId/m3u8/:resolution/index.m3u8', async (c) => {
const shareId = c.req.param('shareId')
const fileId = c.req.param('fileId')
const resolution = c.req.param('resolution')
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
try {
const shareLink = await shareService.verifyPublicAccess(fileId, password)
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
const playlist = await assetService.getHlsVariantPlaylist({
assetId: targetAssetId,
+54 -15
View File
@@ -16,11 +16,14 @@ describe('S3 API', () => {
vi.resetModules()
})
it('GET /files/* calls serveStatic with correct options', async () => {
it('GET /files/* calls serveStatic with correct options when valid signature is provided', async () => {
const { signLocalGetUrl } = await import('@shumai/core/src/s3/s3')
const { signature, expires } = signLocalGetUrl('b1', 'test.webp')
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
const res = await app.request('/files/b1/test.webp')
const res = await app.request(`/files/b1/test.webp?Signature=${signature}&Expires=${expires}`)
expect(res.status).toBe(200)
expect(await res.text()).toBe('static-file')
@@ -39,7 +42,40 @@ describe('S3 API', () => {
}
})
it('GET /files/* returns 403 Forbidden when signature is missing', async () => {
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
const res = await app.request('/files/b1/test.webp')
expect(res.status).toBe(403)
const json = await res.json()
expect(json.error).toContain('Forbidden')
})
it('GET /files/* returns 403 Forbidden when signature is invalid', async () => {
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
const res = await app.request('/files/b1/test.webp?Signature=invalidsig&Expires=9999999999')
expect(res.status).toBe(403)
})
it('GET /files/* returns 403 Forbidden when signature is expired', async () => {
const { signLocalGetUrl } = await import('@shumai/core/src/s3/s3')
// Signed with -10 seconds (expired)
const { signature, expires } = signLocalGetUrl('b1', 'test.webp', -10)
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
const res = await app.request(`/files/b1/test.webp?Signature=${signature}&Expires=${expires}`)
expect(res.status).toBe(403)
})
it('GET /files/* sets Content-Disposition with filename when download=1 & filename query params are present', async () => {
const { signLocalGetUrl } = await import('@shumai/core/src/s3/s3')
const { signature, expires } = signLocalGetUrl('b1', 'test.webp')
// We need to capture the onFound callback from serveStatic options
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let onFoundCallback: ((path: string, c: any) => void) | undefined
@@ -54,7 +90,9 @@ describe('S3 API', () => {
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
await app.request('/files/b1/test.webp?download=1&filename=foo.png')
await app.request(
`/files/b1/test.webp?Signature=${signature}&Expires=${expires}&download=1&filename=foo.png`,
)
expect(onFoundCallback).toBeDefined()
@@ -101,10 +139,7 @@ describe('S3 API', () => {
}
})
it('PUT /files/:bucket/:key calls s3Service.putObject', async () => {
const { s3Service: service } = await import('@shumai/core/src/s3/s3')
const mockPut = vi.spyOn(service, 'putObject').mockResolvedValue(undefined)
it('PUT /files/:bucket/:key returns 404', async () => {
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
const body = new TextEncoder().encode('test-data')
@@ -117,13 +152,17 @@ describe('S3 API', () => {
body,
})
expect(res.status).toBe(200)
expect(await res.text()).toBe('OK')
expect(mockPut).toHaveBeenCalledWith(
'b1',
'test.webp',
expect.any(ReadableStream),
body.byteLength,
)
expect(res.status).toBe(404)
})
it('GET /files/* returns 404 when STORAGE_BACKEND is not local', async () => {
const s3Module = await import('@shumai/core/src/s3/s3')
vi.spyOn(s3Module, 'getStorageBackend').mockReturnValue('s3')
const { default: route } = await import('./s3')
const app = new Hono().route('/files', route)
const res = await app.request('/files/b1/test.webp')
expect(res.status).toBe(404)
})
})
+29 -15
View File
@@ -1,10 +1,38 @@
import { Hono } from 'hono'
import { buildContentDisposition, s3Service } from '@shumai/core/src/s3/s3'
import {
buildContentDisposition,
getStorageBackend,
verifyLocalGetUrlSignature,
} from '@shumai/core/src/s3/s3'
import { serveStatic } from 'hono/bun'
const route = new Hono()
.use('*', async (c, next) => {
if (getStorageBackend() !== 'local') {
return c.notFound()
}
return next()
})
.get(
'/*',
async (c, next) => {
const fullPath = c.req.path.replace(/^\/files\//, '').replace(/^\//, '')
const slashIdx = fullPath.indexOf('/')
if (slashIdx === -1) {
return c.notFound()
}
const bucket = fullPath.slice(0, slashIdx)
const key = fullPath.slice(slashIdx + 1)
const signature = c.req.query('Signature')
const expires = c.req.query('Expires')
if (!verifyLocalGetUrlSignature(bucket, key, signature, expires)) {
return c.json({ error: 'Forbidden: invalid or expired signature' }, 403)
}
return next()
},
serveStatic({
root: './data',
rewriteRequestPath: (path) => path.replace(/^\/files\//, ''),
@@ -15,19 +43,5 @@ const route = new Hono()
},
}),
)
.put('/:bucket/:key{.+}', async (c) => {
const bucket = c.req.param('bucket')
const key = c.req.param('key')
const contentLength = parseInt(c.req.header('content-length') || '0', 10)
try {
const body = c.req.raw.body ?? (await c.req.arrayBuffer())
await s3Service.putObject(bucket, key, body, contentLength)
return c.text('OK')
} catch (e: unknown) {
const error = e as Error
return c.text(error.message || 'Internal Server Error', 500)
}
})
export default route
+109 -4
View File
@@ -7,6 +7,9 @@ import { shareService } from '@shumai/core/src/share/share'
import { assetService } from '@shumai/core/src/asset/asset'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { ShareLinkPasswordInvalidError, ShareLinkExpiredError } from '@shumai/core/src/share/errors'
import { createGuestToken } from '@shumai/core/src/auth/guest-token'
import { userService } from '@shumai/core/src/user/user'
import { notificationService } from '@shumai/core/src/notification/notification'
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
import { watermarkService } from '@shumai/core/src/watermark/watermark'
@@ -121,7 +124,7 @@ describe('Share API', () => {
expect(body.name).toBe('Public Share')
expect(body.hasPassword).toBe(true)
expect(body.allowDownload).toBe(false)
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('folder1', 'pass')
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'folder1', 'pass')
})
test('Unauthorized', async () => {
@@ -206,7 +209,7 @@ describe('Share API', () => {
})
expect(res.status).toBe(200)
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('folder1', 'pass')
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'folder1', 'pass')
expect(assetService.listChildren).toHaveBeenCalledWith(
expect.objectContaining({
assetId: 'folder1',
@@ -989,7 +992,7 @@ describe('Share API', () => {
expect(res.status).toBe(200)
const body = await res.json()
expect(body.url).toBe('https://s3.example.com/download')
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('symlink1', undefined)
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'symlink1', undefined)
expect(assetService.getDownloadUrl).toHaveBeenCalledWith('symlink1', 'files/target1/fly.png')
})
@@ -1033,7 +1036,7 @@ describe('Share API', () => {
expect(res.headers.get('content-type')).toContain('application/vnd.apple.mpegurl')
const text = await res.text()
expect(text).toContain('1080p/index.m3u8?p=secret123')
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('file1', 'secret123')
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'file1', 'secret123')
expect(assetService.getHlsMasterPlaylist).toHaveBeenCalledWith({
assetId: 'file1',
watermarkConfigId: null,
@@ -1093,4 +1096,106 @@ describe('Share API', () => {
})
})
})
describe('POST /shares/:shareId/files/:fileId/comments', () => {
test('rejects comments with attachmentIds on public shares', async () => {
const res = await app.request('/shares/share1/files/file1/comments', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
message: 'hi',
attachmentIds: ['att1'],
}),
})
expect(res.status).toBe(400)
const body = await res.json()
expect(body.error).toContain('Attachments are not allowed')
})
test('rejects spoofed x-guest-user-id header without cookie', async () => {
vi.spyOn(shareService, 'getShareLink').mockResolvedValue({
id: 'share1',
projectId: 'p1',
} as unknown as Awaited<ReturnType<typeof shareService.getShareLink>>)
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue({
id: 'share1',
} as unknown as Awaited<ReturnType<typeof shareService.verifyPublicAccess>>)
vi.spyOn(assetService, 'resolveTargetAssetId').mockResolvedValue('file1')
const res = await app.request('/shares/share1/files/file1/comments', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-guest-user-id': 'victim_user',
},
body: JSON.stringify({ message: 'spoofed comment', attachmentIds: [] }),
})
expect(res.status).toBe(401)
})
test('allows valid guest cookie bound to shareId and creates comment', async () => {
vi.spyOn(shareService, 'getShareLink').mockResolvedValue({
id: 'share1',
projectId: 'p1',
} as unknown as Awaited<ReturnType<typeof shareService.getShareLink>>)
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue({
id: 'share1',
} as unknown as Awaited<ReturnType<typeof shareService.verifyPublicAccess>>)
vi.spyOn(assetService, 'resolveTargetAssetId').mockResolvedValue('file1')
vi.spyOn(userService, 'getUserById').mockResolvedValue({
id: 'guest1',
name: 'Guest Alice',
guestEmail: 'alice@guest.com',
} as unknown as Awaited<ReturnType<typeof userService.getUserById>>)
vi.spyOn(assetService, 'createComment').mockResolvedValue({
id: 'comm1',
message: 'hello from alice',
} as unknown as Awaited<ReturnType<typeof assetService.createComment>>)
vi.spyOn(notificationService, 'create').mockResolvedValue(
undefined as unknown as Awaited<ReturnType<typeof notificationService.create>>,
)
const token = createGuestToken({ userId: 'guest1', shareId: 'share1' })
const res = await app.request('/shares/share1/files/file1/comments', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Cookie: `shumai_guest_share1=${token}`,
},
body: JSON.stringify({ message: 'hello from alice', attachmentIds: [] }),
})
expect(res.status).toBe(201)
expect(assetService.createComment).toHaveBeenCalledWith(
expect.objectContaining({
userId: 'guest1',
message: 'hello from alice',
}),
)
})
test('rejects guest cookie bound to a different shareId', async () => {
vi.spyOn(shareService, 'getShareLink').mockResolvedValue({
id: 'share1',
projectId: 'p1',
} as unknown as Awaited<ReturnType<typeof shareService.getShareLink>>)
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue({
id: 'share1',
} as unknown as Awaited<ReturnType<typeof shareService.verifyPublicAccess>>)
const wrongToken = createGuestToken({ userId: 'guest1', shareId: 'other-share' })
const res = await app.request('/shares/share1/files/file1/comments', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Cookie: `shumai_guest_share1=${wrongToken}`,
},
body: JSON.stringify({ message: 'should fail', attachmentIds: [] }),
})
expect(res.status).toBe(401)
})
})
})
+78 -19
View File
@@ -5,7 +5,6 @@ import teamRoute from './team'
import { authMiddleware } from './middleware/auth'
import { teamService } from '@shumai/core/src/team/team'
import { userMetadataService } from '@shumai/core/src/user-metadata/user-metadata'
import { notificationService } from '@shumai/core/src/notification/notification'
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/core/src/authz/authz', () => ({
@@ -80,14 +79,12 @@ describe('team api', () => {
const app = new Hono().use('*', authMiddleware).route('/', teamRoute)
let mockCreateTeam: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockGetUserTeams: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockJoinTeam: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockGetMe: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockGetTeamMembers: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockGetSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockUpdateSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockListUserMetadata: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockUpsertUserMetadata: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockCreateNotification: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockGetSandboxSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockUpdateSandboxSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
let mockUpdateMe: any // eslint-disable-line @typescript-eslint/no-explicit-any
@@ -95,14 +92,12 @@ describe('team api', () => {
beforeEach(() => {
mockCreateTeam = vi.spyOn(teamService, 'createTeam')
mockGetUserTeams = vi.spyOn(teamService, 'getUserTeams')
mockJoinTeam = vi.spyOn(teamService, 'joinTeam')
mockGetMe = vi.spyOn(teamService, 'getMe')
mockGetTeamMembers = vi.spyOn(teamService, 'getTeamMembers')
mockGetSettings = vi.spyOn(teamService, 'getSettings')
mockUpdateSettings = vi.spyOn(teamService, 'updateSettings')
mockListUserMetadata = vi.spyOn(userMetadataService, 'listMetadata')
mockUpsertUserMetadata = vi.spyOn(userMetadataService, 'upsertMetadata')
mockCreateNotification = vi.spyOn(notificationService, 'create').mockResolvedValue()
mockGetSandboxSettings = vi.spyOn(teamService, 'getSandboxSettings')
mockUpdateSandboxSettings = vi.spyOn(teamService, 'updateSandboxSettings')
mockUpdateMe = vi.spyOn(teamService, 'updateMe')
@@ -148,20 +143,9 @@ describe('team api', () => {
expect(mockGetUserTeams).toHaveBeenCalledWith(expect.objectContaining({ userId: 'user1' }))
})
it('POST /teams/:teamId/members joins team and sends notification', async () => {
mockJoinTeam.mockResolvedValue(undefined)
it('POST /teams/:teamId/members returns 404', async () => {
const res = await app.request('/teams/t1/members', { method: 'POST' })
expect(res.status).toBe(204)
expect(mockJoinTeam).toHaveBeenCalledWith({ teamId: 't1', userId: 'user1' })
expect(mockCreateNotification).toHaveBeenCalledWith(
expect.objectContaining({
type: 'new_user_join_team',
teamId: 't1',
userId: 'user1',
}),
)
expect(res.status).toBe(404)
})
it('GET /teams/:teamId/me returns user role in team', async () => {
@@ -244,7 +228,23 @@ describe('team api', () => {
permission: Permission.Read,
}),
)
expect(mockGetSettings).toHaveBeenCalledWith('t1')
expect(mockGetSettings).toHaveBeenCalledWith('t1', { isAdmin: true })
})
it('GET /teams/:teamId/settings passes isAdmin: false when user lacks admin permission', async () => {
vi.mocked(authzService.hasPermission).mockImplementation(async (params) => {
if (params.permission === Permission.Admin) {
throw new Error('Forbidden')
}
})
mockGetSettings.mockResolvedValue({
archiveStorage: { enabled: true },
})
const res = await app.request('/teams/t1/settings')
expect(res.status).toBe(200)
expect(mockGetSettings).toHaveBeenCalledWith('t1', { isAdmin: false })
})
it('GET /teams/:teamId/settings does not return emailNotification to readers', async () => {
@@ -381,6 +381,65 @@ describe('team api', () => {
expect(mockUpdateSettings).toHaveBeenCalledWith('t1', 'appearance.hideAgent', true)
})
it('PATCH /teams/:teamId/settings updates autoArchive settings object', async () => {
mockUpdateSettings.mockResolvedValue({
autoArchive: {
enabled: true,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
})
const res = await app.request('/teams/t1/settings', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
key: 'autoArchive',
value: {
enabled: true,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
}),
})
expect(res.status).toBe(200)
const data = await res.json()
expect(data.autoArchive.enabled).toBe(true)
expect(data.autoArchive.days).toBe(30)
expect(mockUpdateSettings).toHaveBeenCalledWith('t1', 'autoArchive', {
enabled: true,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
})
})
it('PATCH /teams/:teamId/settings updates autoArchive.enabled individual field', async () => {
mockUpdateSettings.mockResolvedValue({
autoArchive: {
enabled: false,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
})
const res = await app.request('/teams/t1/settings', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
key: 'autoArchive.enabled',
value: false,
}),
})
expect(res.status).toBe(200)
expect(mockUpdateSettings).toHaveBeenCalledWith('t1', 'autoArchive.enabled', false)
})
it('GET /teams/:teamId/user-metadata returns all metadata for the user in the team', async () => {
mockListUserMetadata.mockResolvedValue([
{ key: 'key1', value: 'value1' },
+11 -27
View File
@@ -3,7 +3,6 @@ import { zValidator } from '@hono/zod-validator'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { teamService } from '@shumai/core/src/team/team'
import { userMetadataService } from '@shumai/core/src/user-metadata/user-metadata'
import { notificationService } from '@shumai/core/src/notification/notification'
import {
createTeamRequestSchema,
getUserTeamsRequestSchema,
@@ -31,7 +30,6 @@ import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
import { workflowTaskService } from '@shumai/core/src/workflowTask/workflowTask'
import { turntableService } from '@shumai/core/src/turntable/turntable'
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
import { NotificationType } from '@shumai/db'
import type { Prisma } from '@shumai/db'
type User = Prisma.UserGetPayload<Record<string, never>>
@@ -67,30 +65,6 @@ const route = new Hono<{ Variables: { user: User } }>()
},
})
})
.post('/teams/:teamId/members', async (c) => {
const user = c.get('user')
const teamId = c.req.param('teamId')
await teamService.joinTeam({
teamId,
userId: user.id,
})
await notificationService.create({
type: NotificationType.new_user_join_team,
teamId,
userId: user.id,
})
await auditLogService.logAction({
action: AuditAction.team_member_add,
teamId,
userId: user.id,
itemId: user.id,
})
return new Response(null, { status: 204 })
})
.get('/teams/:teamId/me', async (c) => {
const user = c.get('user')
const teamId = c.req.param('teamId')
@@ -210,7 +184,17 @@ const route = new Hono<{ Variables: { user: User } }>()
id: teamId,
})
const settings = await teamService.getSettings(teamId)
const isAdmin = await authzService
.hasPermission({
user,
permission: Permission.Admin,
type: ResourceType.Team,
id: teamId,
})
.then(() => true)
.catch(() => false)
const settings = await teamService.getSettings(teamId, { isAdmin })
return c.json(settings)
})
.patch(
+13 -1
View File
@@ -1,8 +1,9 @@
import { describe, expect, it, vi, beforeEach } from 'vitest'
import { Hono } from 'hono'
import uploadRoute from './upload'
import uploadRoute, { localUploadRoute } from './upload'
import { authMiddleware } from './middleware/auth'
import { uploadService } from '@shumai/core/src/upload/upload'
import * as s3Module from '@shumai/core/src/s3/s3'
import type { CreateUploadTaskResponse, TaskInfo } from '@shumai/dtos'
import type { PaginatedData } from '@shumai/core/src/pagination'
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
@@ -243,4 +244,15 @@ describe('Upload API', () => {
)
expect(uploadService.cancelUploadTask).toHaveBeenCalledWith('team1', 'user1', 'task1')
})
describe('PUT /upload/local', () => {
it('returns 404 when STORAGE_BACKEND is s3', async () => {
vi.spyOn(s3Module, 'getStorageBackend').mockReturnValue('s3')
const localApp = new Hono().route('/', localUploadRoute)
const res = await localApp.request('/upload/local?bucket=b&key=k&Signature=sig', {
method: 'PUT',
})
expect(res.status).toBe(404)
})
})
})
+5 -1
View File
@@ -14,7 +14,7 @@ import { notificationService } from '@shumai/core/src/notification/notification'
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
import { assetService } from '@shumai/core/src/asset/asset'
import { NotificationType } from '@shumai/db'
import { s3Service, verifyLocalUrlSignature } from '@shumai/core/src/s3/s3'
import { getStorageBackend, s3Service, verifyLocalUrlSignature } from '@shumai/core/src/s3/s3'
import type { Prisma } from '@shumai/db'
type User = Prisma.UserGetPayload<Record<string, never>>
@@ -23,6 +23,10 @@ export const localUploadRoute = new Hono().put(
'/upload/local',
zValidator('query', localUploadQuerySchema),
async (c) => {
if (getStorageBackend() !== 'local') {
return c.text('Not found', 404)
}
const { bucket, key, Signature } = c.req.valid('query')
if (!verifyLocalUrlSignature(bucket, key, Signature)) {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/core",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -217,6 +217,91 @@ describe('download_asset Operation', () => {
})
})
it('rejects downloading original of an archived asset with 409, while allowing proxy: true when a proxy exists', async () => {
const clip = await createFile('archived.mov', {
key: 'files/up-arch/archived.mov',
media: {
duration: 5,
filesize: 512,
frames: 150,
finishedAt: '2026-01-01T00:00:00Z',
metadata: null,
original: null,
imageTranscodes: [],
proxyType: 'video',
videoTranscodes: [
{
resolution: '720p',
key: 'files/up-arch/archived-720p.mp4',
width: 1280,
height: 720,
},
],
},
})
await prisma.asset.update({
where: { id: clip.id },
data: { status: 'archived' },
})
// proxy: false is blocked with 409
await expect(
runOperation(downloadAssetOp, as(owner), { assetIds: [clip.id], proxy: false }),
).rejects.toMatchObject({ status: 409 })
// proxy: true succeeds using the MP4 proxy in main storage
const { files } = await runOperation(downloadAssetOp, as(owner), {
assetIds: [clip.id],
proxy: true,
})
expect(files).toHaveLength(1)
expect(files[0]).toMatchObject({
assetId: clip.id,
kind: 'proxy',
name: 'archived.mp4',
})
})
it('downloads PDF rendition when proxy: true is requested for Office documents with a pdfTranscode', async () => {
const doc = await createFile('presentation.pptx', {
key: 'files/up-office/presentation.pptx',
media: {
duration: 0,
filesize: 1024,
frames: 0,
finishedAt: '2026-01-01T00:00:00Z',
metadata: null,
original: null,
imageTranscodes: [],
videoTranscodes: [],
proxyType: 'pdf',
pdfTranscode: {
key: 'files/up-office/presentation.pdf',
},
},
})
const originalDownload = await runOperation(downloadAssetOp, as(owner), {
assetIds: [doc.id],
proxy: false,
})
expect(originalDownload.files[0]).toMatchObject({
assetId: doc.id,
kind: 'original',
name: 'presentation.pptx',
})
const proxyDownload = await runOperation(downloadAssetOp, as(owner), {
assetIds: [doc.id],
proxy: true,
})
expect(proxyDownload.files[0]).toMatchObject({
assetId: doc.id,
kind: 'proxy',
name: 'presentation.pdf',
})
})
it.each([[[]], [Array.from({ length: 51 }, (_, i) => `a-${i}`)]])(
'rejects %# asset ID lists outside 1 to 50',
async (assetIds) => {
@@ -30,6 +30,9 @@ export function proxyKeyOf(media: PrismaJson.MediaInfo | null): string | undefin
if (media.proxyType === 'video' || media.videoTranscodes?.length || media.videoPreview?.key) {
return media.videoTranscodes?.[0]?.key || media.videoPreview?.key || undefined
}
if (media.pdfTranscode?.key && media.pdfTranscode.key !== media.original?.key) {
return media.pdfTranscode.key
}
return media.original?.key || undefined
}
@@ -66,6 +69,19 @@ export class AssetDownloadService {
})
}
const isOriginal = key === originalKey
const isArchived =
asset.status === 'archiving' ||
asset.status === 'archived' ||
asset.status === 'restoring' ||
Boolean(asset.storageKey?.isArchived)
if (isArchived && (!opts.proxy || isOriginal)) {
throw new HTTPException(409, {
message: opts.proxy
? `Asset ${assetId} ("${asset.name}") is archived and has no proxy file to download. Restore the asset first.`
: `Original file for asset ${assetId} ("${asset.name}") is archived. Restore the asset or request proxy: true to download its proxy.`,
})
}
const name = nameForKey(asset.name || 'file', key, isOriginal)
const bucket = storageBucket()
@@ -244,6 +244,60 @@ describe('AssetReadService', () => {
expect(s3Service.getObject).not.toHaveBeenCalled()
expect(outcome.status === 'ready' && outcome.result.text?.content).toBe('')
})
it('rejects reading raw text or native PDF pages when the original is archived, while allowing proxy.pdf', async () => {
const archivedText = await createFile('notes.md', {
mediaType: 'text/markdown',
key: 'raw/notes.md',
status: 'archived',
})
await expect(
assetReadService.read(
user(owner),
{ assetId: archivedText.id, docConfig: { mode: 'text' } },
now,
),
).rejects.toMatchObject({
status: 400,
message: expect.stringContaining('is archived in cold storage'),
})
const archivedNativePdf = await createFile('report.pdf', {
mediaType: 'application/pdf',
key: 'raw/report.pdf',
status: 'archived',
media: {
proxyType: 'pdf',
pdfTranscode: { key: 'raw/report.pdf', numPages: 3 },
} as unknown as PrismaJson.MediaInfo,
})
await expect(
assetReadService.read(
user(owner),
{ assetId: archivedNativePdf.id, docConfig: { mode: 'pages', startPage: 1, endPage: 1 } },
now,
),
).rejects.toMatchObject({
status: 400,
message: expect.stringContaining('is archived in cold storage'),
})
const archivedOfficeDoc = await createFile('deck.pptx', {
mediaType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
key: 'raw/deck.pptx',
status: 'archived',
media: {
proxyType: 'pdf',
pdfTranscode: { key: 'transcoded/deck/proxy.pdf', numPages: 3 },
} as unknown as PrismaJson.MediaInfo,
})
const officeOutcome = await assetReadService.read(
user(owner),
{ assetId: archivedOfficeDoc.id, docConfig: { mode: 'pages', startPage: 1, endPage: 1 } },
now,
)
expect(officeOutcome.status).toBe('pending')
})
})
describe('render tasks', () => {
@@ -55,6 +55,11 @@ function toReadable(asset: AssetRecord): ReadableAsset {
filenameLower: asset.name?.toLowerCase() || '',
storageKey: asset.storageKey?.key ?? null,
media: asset.media as PrismaJson.MediaInfo | null,
isArchived:
asset.status === 'archiving' ||
asset.status === 'archived' ||
asset.status === 'restoring' ||
Boolean(asset.storageKey?.isArchived),
}
}
@@ -107,9 +107,23 @@ export const documentReader: AssetReader<'document'> = {
`Asset "${asset.name}" is a binary PDF/document and cannot be read as raw text. Please call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ... } to view its visual pages.`,
)
}
if (asset.isArchived) {
throw badRequest(
`Asset "${asset.name}" is archived in cold storage and cannot be read as raw text. Restore the asset first, or use docConfig: { mode: "pages", startPage: 1, endPage: ... } if it has a generated PDF proxy.`,
)
}
return
}
if (
asset.isArchived &&
(!asset.media?.pdfTranscode?.key || asset.media.pdfTranscode.key === asset.storageKey)
) {
throw badRequest(
`Asset "${asset.name}" is a native PDF whose original file is archived in cold storage. Restore the asset first to render its pages.`,
)
}
if (startPage === null || endPage === null) {
throw badRequest('startPage and endPage are required when docConfig mode is "pages".')
}
@@ -1,6 +1,6 @@
import { WorkflowTaskType } from '@shumai/db'
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
import type { AssetReader } from '../types'
import { badRequest, type AssetReader } from '../types'
/** Images (including PSD): the web proxy, or a copy with the annotation drawn on top. */
export const imageReader: AssetReader<'image'> = {
@@ -16,7 +16,13 @@ export const imageReader: AssetReader<'image'> = {
validate() {},
async plan(asset, _options, { annotation }) {
const key = asset.media?.imageTranscodes?.[0]?.key || asset.storageKey
const proxyKey = asset.media?.imageTranscodes?.[0]?.key
if (!proxyKey && asset.isArchived) {
throw badRequest(
`Asset "${asset.name}" is archived in cold storage and has no web image proxy. Restore the asset first.`,
)
}
const key = proxyKey || asset.storageKey
if (!key) {
throw new Error('No media content found for this asset.')
}
+2
View File
@@ -15,6 +15,8 @@ export interface ReadableAsset {
filenameLower: string
storageKey: string | null
media: PrismaJson.MediaInfo | null
/** True when the original raw file is archived, archiving, or restoring. */
isArchived?: boolean
}
export interface ReadAnnotation {
@@ -0,0 +1,517 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import * as fs from 'fs'
import * as path from 'path'
import {
AssetStatus,
AssetType,
prisma,
WorkflowTaskStatus,
WorkflowTaskType,
type User,
} from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { projectService } from '@shumai/core/src/project/project'
import { assetService } from './asset'
import { searchService } from '../search/search'
import { resetArchiveStorageServiceForTests, getArchiveStorageService } from '../s3/archive-storage'
describe('AssetService Archive & Restore', () => {
setupTestDbHooks()
const archiveRoot = path.join(process.cwd(), 'data-test-asset-archive')
let originalEnv: NodeJS.ProcessEnv
let owner: User
let teamId: string
let projectId: string
let rootFolderId: string
beforeEach(async () => {
originalEnv = { ...process.env }
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
resetArchiveStorageServiceForTests()
if (fs.existsSync(archiveRoot)) {
fs.rmSync(archiveRoot, { recursive: true, force: true })
}
owner = await prisma.user.create({
data: { name: 'Archive Owner', email: 'archive-owner@test.com', password: 'pw' },
})
const team = await prisma.team.create({ data: { name: 'Archive Team' } })
teamId = team.id
await prisma.teamMember.create({ data: { teamId, userId: owner.id, role: 'owner' } })
const project = await projectService.createProject(owner, { teamId, name: 'Archive Project' })
projectId = project.id
rootFolderId = project.rootFolder!
})
afterEach(() => {
process.env = originalEnv
resetArchiveStorageServiceForTests()
if (fs.existsSync(archiveRoot)) {
fs.rmSync(archiveRoot, { recursive: true, force: true })
}
})
async function createProcessedFile(params: {
name: string
parentId?: string
key: string
media?: Partial<PrismaJson.MediaInfo>
isArchived?: boolean
status?: AssetStatus
}) {
const asset = await assetService.createAsset({
name: params.name,
type: 'file',
parentId: params.parentId ?? rootFolderId,
projectId,
key: params.key,
sizeByte: 1024,
contentType: 'video/mp4',
creatorId: owner.id,
})
if (params.isArchived) {
await prisma.storageKey.update({
where: { key: params.key },
data: { isArchived: true },
})
}
const fullMedia: PrismaJson.MediaInfo | undefined = params.media
? {
duration: 0,
filesize: 1024,
frames: 0,
finishedAt: '2026-01-01T00:00:00Z',
metadata: null,
original: null,
imageTranscodes: [],
videoTranscodes: [],
...params.media,
}
: undefined
await prisma.asset.update({
where: { id: asset.id },
data: {
status: params.status ?? (params.isArchived ? AssetStatus.archived : AssetStatus.processed),
...(fullMedia ? { media: fullMedia } : {}),
},
})
return prisma.asset.findUniqueOrThrow({
where: { id: asset.id },
include: { storageKey: true },
})
}
it('rejects archive and restore when ARCHIVE_STORAGE_BACKEND is not configured', async () => {
delete process.env.ARCHIVE_STORAGE_BACKEND
resetArchiveStorageServiceForTests()
const file = await createProcessedFile({
name: 'clip.mp4',
key: 'files/01TEST/clip.mp4',
})
await expect(assetService.archiveAssets([file.id])).rejects.toMatchObject({
status: 400,
message: 'Archive storage is not configured',
})
await expect(assetService.restoreArchivedAssets([file.id])).rejects.toMatchObject({
status: 400,
message: 'Archive storage is not configured',
})
})
it('archives and restores standalone files, deduplicating shared StorageKey across copies and updating all copies', async () => {
const file = await createProcessedFile({
name: 'shared.mp4',
key: 'files/01SHARED/shared.mp4',
})
const subFolder = await assetService.createAsset({
name: 'Copies',
type: 'folder',
parentId: rootFolderId,
projectId,
creatorId: owner.id,
})
const [copiedId] = await assetService.copyAssets({
assetIds: [file.id],
newParentId: subFolder.id,
creatorId: owner.id,
withComments: false,
})
// Archive original -> both original and copy share the StorageKey, so both transition to 'archiving' with 1 WorkflowTask
const archiveRes = await assetService.archiveAssets([file.id, copiedId])
expect(archiveRes.archivedAssetIds.sort()).toEqual([file.id, copiedId].sort())
expect(archiveRes.taskIds).toHaveLength(1)
expect(archiveRes.skippedCount).toBe(0)
const updatedRows = await prisma.asset.findMany({
where: { id: { in: [file.id, copiedId] } },
})
expect(updatedRows.map((r) => r.status)).toEqual([AssetStatus.archiving, AssetStatus.archiving])
const task = await prisma.workflowTask.findUniqueOrThrow({
where: { id: archiveRes.taskIds[0] },
})
expect(task.type).toBe(WorkflowTaskType.archive_asset)
expect(task.payload?.archive?.storageKeyId).toBe(file.storageKeyId)
// Simulate workflow completion -> StorageKey.isArchived = true, Asset.status = 'archived'
await prisma.storageKey.update({
where: { id: file.storageKeyId! },
data: { isArchived: true },
})
await prisma.asset.updateMany({
where: { storageKeyId: file.storageKeyId! },
data: { status: AssetStatus.archived },
})
await prisma.workflowTask.update({
where: { id: task.id },
data: { status: WorkflowTaskStatus.completed },
})
// Restore from archive via the copy -> both transition to 'restoring' with 1 restore_asset task
const restoreRes = await assetService.restoreArchivedAssets([copiedId])
expect(restoreRes.restoredAssetIds.sort()).toEqual([file.id, copiedId].sort())
expect(restoreRes.taskIds).toHaveLength(1)
const restoringRows = await prisma.asset.findMany({
where: { id: { in: [file.id, copiedId] } },
})
expect(restoringRows.map((r) => r.status)).toEqual([
AssetStatus.restoring,
AssetStatus.restoring,
])
})
it('recursively archives and restores folders and version stacks without mutating folder or version_stack DB status', async () => {
const folder = await assetService.createAsset({
name: 'Shoot Day 1',
type: 'folder',
parentId: rootFolderId,
projectId,
creatorId: owner.id,
})
const v1 = await createProcessedFile({
name: 'cut_v1.mp4',
parentId: folder.id,
key: 'files/01V1/cut_v1.mp4',
})
const v2 = await createProcessedFile({
name: 'cut_v2.mp4',
parentId: folder.id,
key: 'files/01V2/cut_v2.mp4',
})
// Stack v2 onto v1 to create a version_stack inside folder
await assetService.reparentAssets({
assetIds: [v2.id],
newParentId: v1.id,
creatorId: owner.id,
})
const v1Reloaded = await prisma.asset.findUniqueOrThrow({ where: { id: v1.id } })
const stackId = v1Reloaded.parentId!
const stackRowBefore = await prisma.asset.findUniqueOrThrow({ where: { id: stackId } })
expect(stackRowBefore.type).toBe(AssetType.version_stack)
expect(stackRowBefore.status).toBe(AssetStatus.uploaded)
// Archive the parent folder
const archiveRes = await assetService.archiveAssets([folder.id])
expect(archiveRes.archivedAssetIds.sort()).toEqual([v1.id, v2.id].sort())
expect(archiveRes.taskIds).toHaveLength(2)
// Verify folder and version_stack DB statuses remain 'uploaded'
const folderAfter = await prisma.asset.findUniqueOrThrow({ where: { id: folder.id } })
const stackAfter = await prisma.asset.findUniqueOrThrow({ where: { id: stackId } })
expect(folderAfter.status).toBe(AssetStatus.uploaded)
expect(stackAfter.status).toBe(AssetStatus.uploaded)
// Mark both versions as 'archived' and verify version_stack AssetInfo reflects latestVersion status ('archived')
await prisma.storageKey.updateMany({
where: { id: { in: [v1.storageKeyId!, v2.storageKeyId!] } },
data: { isArchived: true },
})
await prisma.asset.updateMany({
where: { id: { in: [v1.id, v2.id] } },
data: { status: AssetStatus.archived },
})
const stackInfo = await assetService.getAsset({ assetId: stackId })
expect(stackInfo.status).toBe(AssetStatus.archived)
expect(stackInfo.versionStack?.versions.map((v) => v.status)).toEqual([
AssetStatus.archived,
AssetStatus.archived,
])
})
it('omits original.url and native PDF pdfTranscode.url when archived, while keeping Office proxy.pdf url', async () => {
const nativePdf = await createProcessedFile({
name: 'script.pdf',
key: 'files/01PDF/script.pdf',
isArchived: true,
media: {
proxyType: 'pdf',
original: { key: 'files/01PDF/script.pdf', filesizeInBytes: 1024, codec: '' },
pdfTranscode: { key: 'files/01PDF/script.pdf' },
videoTranscodes: [],
imageTranscodes: [],
},
})
const officeDoc = await createProcessedFile({
name: 'deck.pptx',
key: 'files/01PPTX/deck.pptx',
isArchived: true,
media: {
proxyType: 'pdf',
original: { key: 'files/01PPTX/deck.pptx', filesizeInBytes: 2048, codec: '' },
pdfTranscode: { key: 'files/01PPTX/proxy.pdf' },
videoTranscodes: [],
imageTranscodes: [],
},
})
const nativePdfInfo = await assetService.getAsset({ assetId: nativePdf.id })
expect(nativePdfInfo.media?.original?.key).toBe('files/01PDF/script.pdf')
expect(nativePdfInfo.media?.pdfTranscode?.url).toBeUndefined()
const officeDocInfo = await assetService.getAsset({ assetId: officeDoc.id })
expect(officeDocInfo.media?.original?.key).toBe('files/01PPTX/deck.pptx')
expect(officeDocInfo.media?.pdfTranscode?.url).toBeDefined()
})
it('blocks original file download via getDownloadUrl with 409 when archived, while allowing proxy keys, and skips archived files in getDownloadLinks', async () => {
const archivedVideo = await createProcessedFile({
name: 'archived.mp4',
key: 'files/01VID/archived.mp4',
isArchived: true,
media: {
proxyType: 'video',
original: { key: 'files/01VID/archived.mp4', filesizeInBytes: 1024, codec: 'h264' },
videoTranscodes: [
{
resolution: '720p',
key: 'files/01VID/archived-720p.mp4',
width: 1280,
height: 720,
},
],
imageTranscodes: [],
},
})
const hotVideo = await createProcessedFile({
name: 'hot.mp4',
key: 'files/01HOT/hot.mp4',
isArchived: false,
})
// Original key is blocked with 409
await expect(
assetService.getDownloadUrl(archivedVideo.id, 'files/01VID/archived.mp4'),
).rejects.toMatchObject({
status: 409,
})
// Proxy key in same asset folder is allowed
const proxyUrl = await assetService.getDownloadUrl(
archivedVideo.id,
'files/01VID/archived-720p.mp4',
)
expect(proxyUrl).toContain('archived-720p.mp4')
// Batch getDownloadLinks skips archivedVideo and returns hotVideo + skippedArchivedCount = 1
const links = await assetService.getDownloadLinks([archivedVideo.id, hotVideo.id])
expect(links.files).toHaveLength(1)
expect(links.files[0].id).toBe(hotVideo.id)
expect(links.skippedArchivedCount).toBe(1)
})
it('preserves archived status when restoring from Trash and when copying assets, and purges archive storage during GC', async () => {
const archivedFile = await createProcessedFile({
name: 'cold.mp4',
key: 'files/01COLD/cold.mp4',
isArchived: true,
})
// Write physical file into local archive storage
const archiveStorage = getArchiveStorageService()
await archiveStorage.uploadStream(
'files/01COLD/cold.mp4',
new ReadableStream({
start(c) {
c.enqueue(new TextEncoder().encode('cold bytes'))
c.close()
},
}),
10,
'video/mp4',
)
// Copy archived asset -> new copy has status 'archived'
const targetFolder = await assetService.createAsset({
name: 'Target',
type: 'folder',
parentId: rootFolderId,
projectId,
creatorId: owner.id,
})
const [copiedId] = await assetService.copyAssets({
assetIds: [archivedFile.id],
newParentId: targetFolder.id,
creatorId: owner.id,
withComments: false,
})
const copiedRow = await prisma.asset.findUniqueOrThrow({ where: { id: copiedId } })
expect(copiedRow.status).toBe(AssetStatus.archived)
// Delete and restore from Trash -> status returns to 'archived' (not 'processed')
await assetService.deleteAssets([archivedFile.id])
const trashedRow = await prisma.asset.findUniqueOrThrow({ where: { id: archivedFile.id } })
expect(trashedRow.status).toBe(AssetStatus.trashed)
await assetService.restoreAssets([archivedFile.id])
const restoredFromTrash = await prisma.asset.findUniqueOrThrow({
where: { id: archivedFile.id },
})
expect(restoredFromTrash.status).toBe(AssetStatus.archived)
// Filter via SearchService with isArchived condition
const archivedSearch = await searchService.search(rootFolderId, {
operator: 'AND',
conditions: [{ field: 'isArchived', operator: 'eq', value: true }],
recursively: true,
isSemantic: false,
})
expect(archivedSearch.data.map((d) => d.id).sort()).toEqual([archivedFile.id, copiedId].sort())
const unarchivedSearch = await searchService.search(rootFolderId, {
operator: 'AND',
conditions: [{ field: 'isArchived', operator: 'eq', value: false }],
recursively: true,
isSemantic: false,
})
expect(unarchivedSearch.data.map((d) => d.id)).toEqual([])
// Purge both assets and run GC -> file is deleted from archive storage too
await assetService.deleteAssets([archivedFile.id, copiedId])
await prisma.storageKey.update({
where: { id: archivedFile.storageKeyId! },
data: { createdAt: new Date(Date.now() - 60_000) },
})
await assetService.emptyTrash(projectId)
await expect(archiveStorage.headObject('files/01COLD/cold.mp4')).rejects.toThrow(/NoSuchKey/)
const skAfter = await prisma.storageKey.findUnique({
where: { id: archivedFile.storageKeyId! },
})
expect(skAfter).toBeNull()
})
it('refuses to archive when a sibling copy sharing storageKeyId is processing or has an active task', async () => {
const originalFile = await createProcessedFile({
name: 'original.mp4',
key: 'files/01SIB/original.mp4',
})
// Sibling copy sharing the same storageKeyId
const siblingFile = await prisma.asset.create({
data: {
name: 'sibling.mp4',
type: AssetType.file,
status: AssetStatus.processing,
storageKeyId: originalFile.storageKeyId,
projectId,
parentId: rootFolderId,
creatorId: owner.id,
},
})
// Trying to archive originalFile when siblingFile is processing throws 409
await expect(assetService.archiveAssets([originalFile.id])).rejects.toMatchObject({
status: 409,
message: 'Only processed assets can be archived',
})
// Now set sibling to processed, but give it an active workflowTask
await prisma.asset.update({
where: { id: siblingFile.id },
data: { status: AssetStatus.processed },
})
const activeTask = await prisma.workflowTask.create({
data: {
assetId: siblingFile.id,
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.processing,
projectId,
teamId,
},
})
await expect(assetService.archiveAssets([originalFile.id])).rejects.toMatchObject({
status: 409,
message: 'Cannot archive asset while a workflow task is in progress',
})
// Once task completes, archive succeeds
await prisma.workflowTask.update({
where: { id: activeTask.id },
data: { status: WorkflowTaskStatus.completed },
})
const res = await assetService.archiveAssets([originalFile.id])
expect(res.archivedAssetIds.sort()).toEqual([originalFile.id, siblingFile.id].sort())
})
it('ignores symlinks inside folders during recursive folder archiving', async () => {
const folder = await assetService.createAsset({
name: 'archive-folder',
type: 'folder',
parentId: rootFolderId,
projectId,
creatorId: owner.id,
})
const realFile = await createProcessedFile({
name: 'nested-real.mp4',
key: 'files/01NEST/nested-real.mp4',
parentId: folder.id,
})
// External target file
const externalFile = await createProcessedFile({
name: 'external.mp4',
key: 'files/01EXT/external.mp4',
parentId: rootFolderId,
})
// Create a symlink inside the folder pointing to externalFile
await prisma.asset.create({
data: {
name: 'symlink-inside.mp4',
type: AssetType.symlink,
status: AssetStatus.processed,
projectId,
parentId: folder.id,
targetId: externalFile.id,
creatorId: owner.id,
},
})
const res = await assetService.archiveAssets([folder.id])
// Only realFile should be archived, NOT externalFile
expect(res.archivedAssetIds).toEqual([realFile.id])
const extRow = await prisma.asset.findUniqueOrThrow({ where: { id: externalFile.id } })
expect(extRow.status).toBe(AssetStatus.processed)
})
})
+520
View File
@@ -0,0 +1,520 @@
import {
AssetStatus,
AssetType,
Prisma,
prisma,
type PrismaClient,
WorkflowTaskStatus,
WorkflowTaskType,
} from '@shumai/db'
import type { ArchiveAssetsResponse, RestoreArchivedAssetsResponse } from '@shumai/dtos'
import { HTTPException } from 'hono/http-exception'
import { isArchiveStorageEnabled } from '@shumai/core/src/s3/archive-storage'
export class AssetArchiveService {
constructor(private readonly prismaClient: PrismaClient = prisma) {}
async resolveCandidateFileAssetsForArchiveOrRestore(assetIds: string[]) {
if (assetIds.length === 0) return []
const topAssets = await this.prismaClient.asset.findMany({
where: { id: { in: assetIds }, isDeleted: false },
select: { id: true, type: true },
})
const foundIds = new Set(topAssets.map((a) => a.id))
const uniqueRequested = Array.from(new Set(assetIds))
if (foundIds.size !== uniqueRequested.length) {
throw new HTTPException(404, { message: 'Asset not found' })
}
if (topAssets.some((a) => a.type === AssetType.symlink)) {
throw new HTTPException(400, { message: 'Symlinks cannot be archived or restored' })
}
const startingIds = topAssets.map((a) => a.id)
const expandIdsRecursively = async (rootIds: string[]) => {
if (rootIds.length === 0) return []
return this.prismaClient.$queryRaw<
Array<{
id: string
type: string
}>
>`
WITH RECURSIVE descendant AS (
SELECT id, type::text AS type, is_deleted
FROM assets
WHERE id IN (${Prisma.join(rootIds)}) AND is_deleted = false AND type != 'symlink'
UNION ALL
SELECT a.id, a.type::text AS type, a.is_deleted
FROM assets a
INNER JOIN descendant d ON a.parent_id = d.id
WHERE a.is_deleted = false AND a.type != 'symlink'
)
SELECT id, type FROM descendant;
`
}
const descendants = await expandIdsRecursively(startingIds)
const fileIds = Array.from(
new Set(descendants.filter((r) => r.type === AssetType.file).map((r) => r.id)),
)
if (fileIds.length === 0) {
return []
}
return this.prismaClient.asset.findMany({
where: {
id: { in: fileIds },
isDeleted: false,
type: AssetType.file,
status: {
notIn: [AssetStatus.trashed, AssetStatus.pending_purge],
},
},
include: {
storageKey: true,
project: { select: { id: true, teamId: true } },
},
orderBy: { id: 'asc' },
})
}
async archiveAssets(
assetIds: string[],
options?: {
suppressBusyErrors?: boolean
autoArchivePolicy?: {
method: 'not_updated' | 'after_created'
cutoffDate: Date
restoreCutoffDate: Date
}
},
): Promise<ArchiveAssetsResponse> {
if (!isArchiveStorageEnabled()) {
throw new HTTPException(400, { message: 'Archive storage is not configured' })
}
if (assetIds.length === 0) {
return { archivedAssetIds: [], taskIds: [], skippedCount: 0 }
}
const candidateFiles = await this.resolveCandidateFileAssetsForArchiveOrRestore(assetIds)
if (candidateFiles.length === 0) {
return { archivedAssetIds: [], taskIds: [], skippedCount: 0 }
}
const candidateStorageKeyIds = Array.from(
new Set(candidateFiles.map((f) => f.storageKeyId).filter((id): id is string => !!id)),
)
// Sibling copies sharing the same storageKeyId must also be protected
const allFilesForKeys = await this.prismaClient.asset.findMany({
where: {
storageKeyId: { in: candidateStorageKeyIds },
isDeleted: false,
type: AssetType.file,
status: { notIn: [AssetStatus.trashed, AssetStatus.pending_purge] },
},
select: { id: true, storageKeyId: true, status: true },
})
const allFileIds = allFilesForKeys.map((f) => f.id)
// Check active transcode/watermark tasks across candidate files AND all siblings sharing storage keys
const activeTasks = await this.prismaClient.workflowTask.findMany({
where: {
assetId: { in: allFileIds },
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
type: {
in: [
WorkflowTaskType.transcode,
WorkflowTaskType.transcode_video,
WorkflowTaskType.transcode_image,
WorkflowTaskType.transcode_pdf,
WorkflowTaskType.transcode_pdf_pages,
WorkflowTaskType.transcode_screenshot,
WorkflowTaskType.transcode_image_annotation,
WorkflowTaskType.transcode_watermark,
],
},
},
select: { assetId: true },
})
const busyAssetIds = new Set(activeTasks.map((t) => t.assetId))
const storageKeysWithBusyTask = new Set<string>()
const storageKeysWithUnprocessedSiblings = new Set<string>()
for (const f of allFilesForKeys) {
if (!f.storageKeyId) continue
if (busyAssetIds.has(f.id)) {
storageKeysWithBusyTask.add(f.storageKeyId)
}
if (f.status === AssetStatus.uploading || f.status === AssetStatus.processing) {
storageKeysWithUnprocessedSiblings.add(f.storageKeyId)
}
}
const eligibleByStorageKey = new Map<string, typeof candidateFiles>()
let skippedCount = 0
let alreadyArchivedCount = 0
let invalidStateCount = 0
let busyCount = 0
for (const file of candidateFiles) {
if (!file.storageKeyId || !file.storageKey || !file.projectId || !file.project?.teamId) {
skippedCount++
continue
}
if (
file.status === AssetStatus.archiving ||
file.status === AssetStatus.archived ||
file.storageKey.isArchived
) {
skippedCount++
alreadyArchivedCount++
continue
}
if (busyAssetIds.has(file.id) || storageKeysWithBusyTask.has(file.storageKeyId)) {
skippedCount++
busyCount++
continue
}
if (
file.status !== AssetStatus.processed ||
storageKeysWithUnprocessedSiblings.has(file.storageKeyId)
) {
skippedCount++
invalidStateCount++
continue
}
const list = eligibleByStorageKey.get(file.storageKeyId) || []
list.push(file)
eligibleByStorageKey.set(file.storageKeyId, list)
}
if (eligibleByStorageKey.size === 0) {
if (!options?.suppressBusyErrors) {
if (alreadyArchivedCount === 0 && busyCount > 0) {
throw new HTTPException(409, {
message: 'Cannot archive asset while a workflow task is in progress',
})
}
if (alreadyArchivedCount === 0 && invalidStateCount > 0) {
throw new HTTPException(409, {
message: 'Only processed assets can be archived',
})
}
}
return { archivedAssetIds: [], taskIds: [], skippedCount }
}
const archivedAssetIdsSet = new Set<string>()
const taskIds: string[] = []
for (const [storageKeyId, filesForKey] of eligibleByStorageKey.entries()) {
const primaryFile = filesForKey[0]
await this.prismaClient.$transaction(async (tx) => {
// 1. Lock the storageKey row to serialize concurrent archive requests
const lockedKeys = await tx.$queryRaw<Array<{ id: string; isArchived: boolean }>>`
SELECT id, is_archived AS "isArchived"
FROM storage_keys
WHERE id = ${storageKeyId}
FOR UPDATE
`
if (!lockedKeys || lockedKeys.length === 0 || lockedKeys[0].isArchived) {
return
}
// If an auto-archive policy was specified, re-verify under lock that all sibling assets meet the age policy
if (options?.autoArchivePolicy) {
const policy = options.autoArchivePolicy
const ineligible = await tx.asset.findFirst({
where: {
storageKeyId,
isDeleted: false,
OR: [
{ type: { not: AssetType.file } },
{ status: { not: AssetStatus.processed } },
...(policy.method === 'not_updated'
? [{ updatedAt: { gt: policy.cutoffDate } }]
: [
{
OR: [
{ createdAt: { gt: policy.cutoffDate } },
{ updatedAt: { gt: policy.restoreCutoffDate } },
],
},
]),
],
},
select: { id: true },
})
if (ineligible) {
return
}
}
// 2. Atomic Filter + Update: only files that are currently 'processed' transition to 'archiving'
const updatedAssets = await tx.$queryRaw<Array<{ id: string }>>`
UPDATE assets
SET status = 'archiving'::"AssetStatus", updated_at = NOW()
WHERE storage_key_id = ${storageKeyId}
AND type = 'file'::"AssetType"
AND status = 'processed'::"AssetStatus"
AND is_deleted = false
RETURNING id
`
if (!updatedAssets || updatedAssets.length === 0) {
return
}
for (const row of updatedAssets) {
archivedAssetIdsSet.add(row.id)
}
const existingTask = await tx.workflowTask.findFirst({
where: {
assetId: { in: filesForKey.map((f) => f.id) },
type: WorkflowTaskType.archive_asset,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
select: { id: true },
})
if (existingTask) {
taskIds.push(existingTask.id)
} else {
const createdTask = await tx.workflowTask.create({
data: {
assetId: primaryFile.id,
type: WorkflowTaskType.archive_asset,
status: WorkflowTaskStatus.pending,
teamId: primaryFile.project!.teamId,
projectId: primaryFile.projectId!,
payload: {
projectId: primaryFile.projectId!,
archive: { storageKeyId },
},
},
})
taskIds.push(createdTask.id)
}
})
}
return {
archivedAssetIds: Array.from(archivedAssetIdsSet),
taskIds,
skippedCount,
}
}
async restoreArchivedAssets(assetIds: string[]): Promise<RestoreArchivedAssetsResponse> {
if (!isArchiveStorageEnabled()) {
throw new HTTPException(400, { message: 'Archive storage is not configured' })
}
if (assetIds.length === 0) {
return { restoredAssetIds: [], taskIds: [], skippedCount: 0 }
}
const candidateFiles = await this.resolveCandidateFileAssetsForArchiveOrRestore(assetIds)
if (candidateFiles.length === 0) {
return { restoredAssetIds: [], taskIds: [], skippedCount: 0 }
}
const candidateStorageKeyIds = Array.from(
new Set(candidateFiles.map((f) => f.storageKeyId).filter((id): id is string => !!id)),
)
const allFilesForKeys = await this.prismaClient.asset.findMany({
where: {
storageKeyId: { in: candidateStorageKeyIds },
isDeleted: false,
type: AssetType.file,
status: { notIn: [AssetStatus.trashed, AssetStatus.pending_purge] },
},
select: { id: true, storageKeyId: true, status: true },
})
const storageKeysWithArchivingSibling = new Set<string>()
for (const f of allFilesForKeys) {
if (!f.storageKeyId) continue
if (f.status === AssetStatus.archiving) {
storageKeysWithArchivingSibling.add(f.storageKeyId)
}
}
const eligibleByStorageKey = new Map<string, typeof candidateFiles>()
let skippedCount = 0
let alreadyRestoringOrHotCount = 0
let currentlyArchivingCount = 0
for (const file of candidateFiles) {
if (!file.storageKeyId || !file.storageKey || !file.projectId || !file.project?.teamId) {
skippedCount++
continue
}
if (
file.status === AssetStatus.archiving ||
storageKeysWithArchivingSibling.has(file.storageKeyId)
) {
skippedCount++
currentlyArchivingCount++
continue
}
if (file.status === AssetStatus.restoring) {
skippedCount++
alreadyRestoringOrHotCount++
continue
}
if (file.status !== AssetStatus.archived && !file.storageKey.isArchived) {
skippedCount++
alreadyRestoringOrHotCount++
continue
}
const list = eligibleByStorageKey.get(file.storageKeyId) || []
list.push(file)
eligibleByStorageKey.set(file.storageKeyId, list)
}
if (eligibleByStorageKey.size === 0) {
if (alreadyRestoringOrHotCount === 0 && currentlyArchivingCount > 0) {
throw new HTTPException(409, {
message: 'Cannot restore asset while it is currently archiving',
})
}
return { restoredAssetIds: [], taskIds: [], skippedCount }
}
const restoredAssetIdsSet = new Set<string>()
const taskIds: string[] = []
for (const [storageKeyId, filesForKey] of eligibleByStorageKey.entries()) {
const primaryFile = filesForKey[0]
await this.prismaClient.$transaction(async (tx) => {
const sharedAssets = await tx.asset.findMany({
where: {
storageKeyId,
isDeleted: false,
type: AssetType.file,
status: AssetStatus.archived,
},
select: { id: true },
})
const sharedAssetIds = sharedAssets.map((a) => a.id)
if (sharedAssetIds.length > 0) {
await tx.asset.updateMany({
where: { id: { in: sharedAssetIds } },
data: { status: AssetStatus.restoring },
})
for (const id of sharedAssetIds) {
restoredAssetIdsSet.add(id)
}
}
const existingTask = await tx.workflowTask.findFirst({
where: {
assetId: { in: filesForKey.map((f) => f.id) },
type: WorkflowTaskType.restore_asset,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
select: { id: true },
})
if (existingTask) {
taskIds.push(existingTask.id)
} else {
const createdTask = await tx.workflowTask.create({
data: {
assetId: primaryFile.id,
type: WorkflowTaskType.restore_asset,
status: WorkflowTaskStatus.pending,
teamId: primaryFile.project!.teamId,
projectId: primaryFile.projectId!,
payload: {
projectId: primaryFile.projectId!,
archive: { storageKeyId },
},
},
})
taskIds.push(createdTask.id)
}
})
}
return {
restoredAssetIds: Array.from(restoredAssetIdsSet),
taskIds,
skippedCount,
}
}
}
export const assetArchiveService = new AssetArchiveService()
/**
* Reverts assets referencing a StorageKey back to their baseline status (archived vs processed)
* when an archive or restore task fails or recovers.
*/
export async function revertArchiveStateForStorageKey(
storageKeyId: string,
client: PrismaClient | Prisma.TransactionClient = prisma,
): Promise<void> {
const sk = await client.storageKey.findUnique({
where: { id: storageKeyId },
select: { isArchived: true },
})
if (!sk) return
const fallbackStatus = sk.isArchived ? AssetStatus.archived : AssetStatus.processed
await client.asset.updateMany({
where: {
storageKeyId,
isDeleted: false,
deletedAt: null,
type: AssetType.file,
status: { in: [AssetStatus.archiving, AssetStatus.restoring] },
},
data: { status: fallbackStatus },
})
}
/**
* Ensures an asset's storage key is not archived before allowing direct download of the original raw file.
*/
export async function assertStorageKeyNotArchived(
key: string,
client: PrismaClient | Prisma.TransactionClient = prisma,
): Promise<void> {
const sk = await client.storageKey.findUnique({
where: { key },
include: {
assets: {
where: { isDeleted: false },
select: { status: true },
},
},
})
if (
sk &&
(sk.isArchived ||
sk.assets.some(
(a) =>
a.status === AssetStatus.archiving ||
a.status === AssetStatus.archived ||
a.status === AssetStatus.restoring,
))
) {
throw new Error(
`Original file for storage key "${key}" is archived. Restore the asset first or download its proxy using assetIds with proxy: true.`,
)
}
}
+251 -6
View File
@@ -4,7 +4,8 @@ import { setupTestDbHooks } from '@shumai/db/test'
import { HTTPException } from 'hono/http-exception'
import { AssetType, AssetStatus, Prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { s3Service, getStorageBackend } from '@shumai/core/src/s3/s3'
import * as archiveStorageModule from '@shumai/core/src/s3/archive-storage'
import { uploadService } from '@shumai/core/src/upload/upload'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { AssetService } from './asset'
@@ -20,7 +21,9 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
deleteObject: vi.fn().mockResolvedValue(1),
deletePrefix: vi.fn().mockResolvedValue(1),
copyObject: vi.fn().mockResolvedValue(undefined),
getStoragePath: vi.fn((bucket: string, key: string) => `/mock/data/${bucket}/${key}`),
},
getStorageBackend: vi.fn().mockReturnValue('local'),
}))
vi.mock('@shumai/core/src/s3/preview-converter', async (importOriginal) => {
@@ -3010,7 +3013,7 @@ describe('AssetService — natural sort by name', () => {
// 4. file1.txt (from symlinkToFile target resolution, deduplicated)
// 5. version2.txt (from symlinkToStack target resolution, deduplicated)
const fileNames = result.map((r) => r.name)
const fileNames = result.files.map((r) => r.name)
expect(fileNames).toHaveLength(3) // 3 unique files
expect(fileNames).toContain('file1.txt')
expect(fileNames).toContain('file2.txt')
@@ -3021,7 +3024,7 @@ describe('AssetService — natural sort by name', () => {
expect(fileNames).not.toContain('stack')
// Check urls are returned correctly (mock s3 presign resolves to 'http://mock-s3-url')
for (const item of result) {
for (const item of result.files) {
expect(item.url).toBe('http://mock-s3-url')
}
@@ -3070,9 +3073,251 @@ describe('AssetService — natural sort by name', () => {
})
const result = await assetService.getDownloadLinks([symlink.id])
expect(result).toHaveLength(1)
expect(result[0].name).toBe('file1.txt')
expect(result[0].id).toBe(file1.id)
expect(result.files).toHaveLength(1)
expect(result.files[0].name).toBe('file1.txt')
expect(result.files[0].id).toBe(file1.id)
})
})
describe('getStoragePaths', () => {
it('returns empty array for empty inputs', async () => {
const result = await assetService.getStoragePaths([])
expect(result).toEqual({ paths: [] })
})
it('resolves files, symlinks, and version stacks while skipping folders', async () => {
const user = await prisma.user.create({
data: {
name: 'StoragePathUser',
email: `storage-path-${Date.now()}@example.com`,
},
})
const team = await prisma.team.create({ data: { name: 'StoragePathTeam' } })
const project = await prisma.project.create({
data: { name: 'StoragePathProject', teamId: team.id },
})
const folder = await prisma.asset.create({
data: {
name: 'MyFolder',
type: AssetType.folder,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.processed,
},
})
const file = await prisma.asset.create({
data: {
name: 'clip.mp4',
type: AssetType.file,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.processed,
storageKey: { create: { key: 'files/ulid1/clip.mp4' } },
},
})
const symlink = await prisma.asset.create({
data: {
name: 'clip-symlink',
type: AssetType.symlink,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.uploaded,
target: { connect: { id: file.id } },
},
})
const stack = await prisma.asset.create({
data: {
name: 'stack.png',
type: AssetType.version_stack,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.processed,
},
})
await prisma.asset.create({
data: {
name: 'stack_v1.png',
type: AssetType.file,
project: { connect: { id: project.id } },
parent: { connect: { id: stack.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.processed,
sortIndex: 'b',
storageKey: { create: { key: 'files/ulid2/stack_v1.png' } },
},
})
const latestChild = await prisma.asset.create({
data: {
name: 'stack_v2.png',
type: AssetType.file,
project: { connect: { id: project.id } },
parent: { connect: { id: stack.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.processed,
sortIndex: 'a',
storageKey: { create: { key: 'files/ulid3/stack_v2.png' } },
},
})
const result = await assetService.getStoragePaths([folder.id, file.id, symlink.id, stack.id])
// folder should be skipped, symlink deduplicated to file, stack resolved to latest child
expect(result.paths).toHaveLength(2)
const names = result.paths.map((p) => p.name)
expect(names).toContain('clip.mp4')
expect(names).toContain('stack_v2.png')
const clipPath = result.paths.find((p) => p.name === 'clip.mp4')!
expect(clipPath.path).toContain('files/ulid1/clip.mp4')
expect(clipPath.isArchived).toBe(false)
const stackPath = result.paths.find((p) => p.name === 'stack_v2.png')!
expect(stackPath.id).toBe(latestChild.id)
expect(stackPath.path).toContain('files/ulid3/stack_v2.png')
})
it('returns S3 path when main storage backend is s3', async () => {
vi.mocked(getStorageBackend).mockReturnValueOnce('s3')
vi.mocked(s3Service.getStoragePath).mockReturnValueOnce('s3://shumai/files/test/s3-file.mp4')
const project = await prisma.project.create({
data: { name: 'P-S3', team: { create: { name: 'T-S3' } } },
})
const user = await prisma.user.create({
data: { name: 'U-S3', email: `s3-${Date.now()}@example.com` },
})
const file = await prisma.asset.create({
data: {
name: 's3-file.mp4',
type: AssetType.file,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.processed,
storageKey: { create: { key: 'files/test/s3-file.mp4' } },
},
})
const res = await assetService.getStoragePaths([file.id])
expect(res.paths).toEqual([
{
id: file.id,
name: 's3-file.mp4',
path: 's3://shumai/files/test/s3-file.mp4',
backend: 's3',
isArchived: false,
},
])
})
it('returns archive S3 path when asset is archived and archive backend is s3', async () => {
vi.spyOn(archiveStorageModule, 'getArchiveStoragePath').mockReturnValueOnce({
path: 's3://my-archive-bucket/files/archived/clip.mp4',
backend: 's3',
})
const project = await prisma.project.create({
data: { name: 'P-Archive-S3', team: { create: { name: 'T-Archive-S3' } } },
})
const user = await prisma.user.create({
data: { name: 'U-Arch', email: `arch-${Date.now()}@example.com` },
})
const file = await prisma.asset.create({
data: {
name: 'clip.mp4',
type: AssetType.file,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.archived,
storageKey: { create: { key: 'files/archived/clip.mp4', isArchived: true } },
},
})
const res = await assetService.getStoragePaths([file.id])
expect(res.paths).toEqual([
{
id: file.id,
name: 'clip.mp4',
path: 's3://my-archive-bucket/files/archived/clip.mp4',
backend: 's3',
isArchived: true,
},
])
})
it('returns archive local path when asset is archived and archive backend is local', async () => {
vi.spyOn(archiveStorageModule, 'getArchiveStoragePath').mockReturnValueOnce({
path: '/mnt/archive/files/archived/clip.mp4',
backend: 'local',
})
const project = await prisma.project.create({
data: { name: 'P-Archive-Local', team: { create: { name: 'T-Archive-Local' } } },
})
const user = await prisma.user.create({
data: { name: 'U-ArchLocal', email: `archloc-${Date.now()}@example.com` },
})
const file = await prisma.asset.create({
data: {
name: 'clip.mp4',
type: AssetType.file,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.archived,
storageKey: { create: { key: 'files/archived/clip.mp4', isArchived: true } },
},
})
const res = await assetService.getStoragePaths([file.id])
expect(res.paths).toEqual([
{
id: file.id,
name: 'clip.mp4',
path: '/mnt/archive/files/archived/clip.mp4',
backend: 'local',
isArchived: true,
},
])
})
it('marks isArchived: true with main storage path when archive storage is disabled', async () => {
vi.spyOn(archiveStorageModule, 'getArchiveStoragePath').mockReturnValueOnce(null)
vi.mocked(s3Service.getStoragePath).mockReturnValueOnce(
'/mock/data/shumai/files/archived/clip.mp4',
)
const project = await prisma.project.create({
data: { name: 'P-DisabledArchive', team: { create: { name: 'T-DisabledArchive' } } },
})
const user = await prisma.user.create({
data: { name: 'U-DisArch', email: `disarch-${Date.now()}@example.com` },
})
const file = await prisma.asset.create({
data: {
name: 'clip.mp4',
type: AssetType.file,
project: { connect: { id: project.id } },
creator: { connect: { id: user.id } },
status: AssetStatus.archived,
storageKey: { create: { key: 'files/archived/clip.mp4', isArchived: true } },
},
})
const res = await assetService.getStoragePaths([file.id])
expect(res.paths).toEqual([
{
id: file.id,
name: 'clip.mp4',
path: '/mock/data/shumai/files/archived/clip.mp4',
backend: 'local',
isArchived: true,
},
])
})
})
+340 -30
View File
@@ -1,6 +1,7 @@
import { cancelWorkflowTask, prisma } from '@shumai/db'
import {
AncestorFolder,
ArchiveAssetsResponse,
AssetInfo,
AttachmentInfo,
ChildPreview,
@@ -14,6 +15,7 @@ import {
ListChildrenRequest,
PreviewInfo,
ReparentAssetsRequest,
RestoreArchivedAssetsResponse,
UpdateAssetNameRequest,
UpdateAssetOrderRequest,
AssetUserInfo,
@@ -21,6 +23,8 @@ import {
CommentExportFormat,
PreviewFormat,
TRASH_RETENTION_DAYS,
GetStoragePathsResponse,
StoragePathItem,
} from '@shumai/dtos'
import {
exportCommentsToFormat,
@@ -41,7 +45,13 @@ import {
import { HTTPException } from 'hono/http-exception'
import { logger } from '@shumai/core/src/logger'
import { PaginatedData, paginateQuery, PaginationParams } from '@shumai/core/src/pagination'
import { s3Service } from '@shumai/core/src/s3/s3'
import { s3Service, getStorageBackend } from '@shumai/core/src/s3/s3'
import {
getArchiveStorageService,
isArchiveStorageEnabled,
getArchiveStoragePath,
} from '@shumai/core/src/s3/archive-storage'
import { assetArchiveService } from './asset-archive'
import {
ensureJpegInStorage,
getJpegKeyForWebp,
@@ -409,7 +419,7 @@ export class AssetService {
const assetsToCopy = await tx.asset.findMany({
where: { id: { in: req.assetIds } },
include: { project: { include: { team: true } } },
include: { project: { include: { team: true } }, storageKey: true },
})
if (assetsToCopy.length !== req.assetIds.length) {
@@ -488,6 +498,17 @@ export class AssetService {
creatorId?: string,
withComments: boolean = false,
): Promise<Asset> {
if (asset.status === AssetStatus.archiving || asset.status === AssetStatus.restoring) {
throw new HTTPException(409, {
message: 'Cannot copy an asset while it is being archived or restored',
})
}
const copiedStatus =
asset.type === AssetType.file && asset.storageKey?.isArchived
? AssetStatus.archived
: asset.status
const newAsset = await tx.asset.create({
data: {
name: asset.name,
@@ -496,7 +517,7 @@ export class AssetService {
mediaType: asset.mediaType,
fileCount: asset.fileCount,
sizeByte: asset.sizeByte,
status: asset.status,
status: copiedStatus,
transcodeTaskId: asset.transcodeTaskId,
media: asset.media || undefined,
isDeleted: asset.isDeleted,
@@ -515,9 +536,10 @@ export class AssetService {
await this.copyComments(tx, asset.id, newAsset.id, projectId, creatorId)
}
if (asset.type === AssetType.folder) {
if (asset.type === AssetType.folder || asset.type === AssetType.version_stack) {
const children = await tx.asset.findMany({
where: { parentId: asset.id },
include: { storageKey: true },
orderBy: { sortIndex: 'asc' },
})
for (const child of children) {
@@ -1229,6 +1251,75 @@ export class AssetService {
return currentId
}
async resolveLatestVersionIds(assetIds: string[]): Promise<string[]> {
if (assetIds.length === 0) return []
const initialAssets = await this.prismaClient.asset.findMany({
where: { id: { in: assetIds }, isDeleted: false },
select: { id: true, type: true, targetId: true },
})
const assetMap = new Map(initialAssets.map((a) => [a.id, a]))
// Find symlinks to resolve
const symlinks = initialAssets.filter((a) => a.type === AssetType.symlink && a.targetId)
const targetIds = Array.from(new Set(symlinks.map((s) => s.targetId!)))
const symlinkTargetMap = new Map<string, { id: string; type: AssetType }>()
if (targetIds.length > 0) {
const targets = await this.prismaClient.asset.findMany({
where: { id: { in: targetIds }, isDeleted: false },
select: { id: true, type: true },
})
for (const t of targets) {
symlinkTargetMap.set(t.id, t)
}
}
// Determine current ID and type for each input ID
const intermediate = assetIds.map((id) => {
const asset = assetMap.get(id)
if (!asset) return { originalId: id, currentId: id, type: null }
if (asset.type === AssetType.symlink && asset.targetId) {
const target = symlinkTargetMap.get(asset.targetId)
if (target) {
return { originalId: id, currentId: target.id, type: target.type }
}
}
return { originalId: id, currentId: asset.id, type: asset.type }
})
// Find version stacks to resolve to their latest child
const versionStackIds = Array.from(
new Set(
intermediate
.filter((item) => item.type === AssetType.version_stack)
.map((item) => item.currentId),
),
)
const latestChildByParent = new Map<string, string>()
if (versionStackIds.length > 0) {
const children = await this.prismaClient.asset.findMany({
where: { parentId: { in: versionStackIds }, isDeleted: false },
select: { id: true, parentId: true, sortIndex: true },
orderBy: { sortIndex: 'asc' },
})
for (const child of children) {
if (child.parentId && !latestChildByParent.has(child.parentId)) {
latestChildByParent.set(child.parentId, child.id)
}
}
}
return intermediate.map((item) => {
if (item.type === AssetType.version_stack) {
const childId = latestChildByParent.get(item.currentId)
if (childId) return childId
}
return item.currentId
})
}
async updateAssetName(req: UpdateAssetNameRequest): Promise<AssetInfo> {
const existing = await this.prismaClient.asset.findUnique({
where: { id: req.id },
@@ -1605,7 +1696,9 @@ export class AssetService {
// 1. Find storage keys with no associated assets and older than 24 hours
// using FOR UPDATE SKIP LOCKED to ensure multiple servers don't pick up the same keys.
const lockedKeys = await this.prismaClient.$queryRaw<{ id: string; key: string }[]>`
const lockedKeys = await this.prismaClient.$queryRaw<
{ id: string; key: string; isArchived: boolean }[]
>`
WITH to_purge AS (
SELECT id FROM storage_keys sk
WHERE NOT EXISTS (SELECT 1 FROM assets a WHERE a.storage_key_id = sk.id)
@@ -1616,17 +1709,30 @@ export class AssetService {
)
UPDATE storage_keys SET status = 'purging', updated_at = NOW()
WHERE id IN (SELECT id FROM to_purge)
RETURNING id, key;
RETURNING id, key, is_archived AS "isArchived";
`
if (lockedKeys.length === 0) return
let physicalFilesDeleted = 0
let purgedCount = 0
const archiveEnabled = isArchiveStorageEnabled()
for (const sk of lockedKeys) {
try {
// Physically delete from S3
if (sk.isArchived && !archiveEnabled) {
logger.warn(
{ storageKeyId: sk.id, key: sk.key },
'Skipping purge of archived storage key because archive storage is not configured',
)
await this.prismaClient.storageKey.update({
where: { id: sk.id },
data: { status: 'active' },
})
continue
}
// Physically delete proxies (and unarchived original) from main storage
const parts = sk.key.split('/')
if (parts.length > 2) {
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
@@ -1637,6 +1743,15 @@ export class AssetService {
physicalFilesDeleted += count
}
// Also delete original raw file from archive storage when archive storage is separate
if (archiveEnabled) {
const archiveStorage = getArchiveStorageService()
if (!archiveStorage.isSameS3Target) {
const archiveDeleted = await archiveStorage.deleteObject(sk.key)
physicalFilesDeleted += archiveDeleted
}
}
// Delete from database
await this.prismaClient.storageKey.delete({
where: { id: sk.id },
@@ -1663,7 +1778,7 @@ export class AssetService {
await this.prismaClient.$transaction(async (tx) => {
const a = await tx.asset.findUnique({
where: { id },
include: { project: { include: { team: true } } },
include: { project: { include: { team: true } }, storageKey: true },
})
if (!a) throw new Error('Asset not found')
if (!a.parentId) throw new Error('Asset has no parent')
@@ -1687,7 +1802,25 @@ export class AssetService {
if (descendantIds.length > 0) {
await tx.asset.updateMany({
where: { id: { in: descendantIds } },
data: { isDeleted: false },
data: { isDeleted: false, deletedAt: null },
})
await tx.asset.updateMany({
where: {
id: { in: descendantIds },
type: AssetType.file,
status: AssetStatus.trashed,
storageKey: { isArchived: true },
},
data: { status: AssetStatus.archived },
})
await tx.asset.updateMany({
where: {
id: { in: descendantIds },
type: AssetType.file,
status: AssetStatus.trashed,
storageKey: { isArchived: false },
},
data: { status: AssetStatus.processed },
})
}
@@ -1724,6 +1857,16 @@ export class AssetService {
}
}
const activeArchiveTask = await tx.workflowTask.findFirst({
where: {
assetId: a.id,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
type: {
in: [WorkflowTaskType.archive_asset, WorkflowTaskType.restore_asset],
},
},
})
const activeTask = await tx.workflowTask.findFirst({
where: {
assetId: a.id,
@@ -1743,11 +1886,22 @@ export class AssetService {
},
})
let restoredStatus: AssetStatus = AssetStatus.processed
if (activeArchiveTask?.type === WorkflowTaskType.archive_asset) {
restoredStatus = AssetStatus.archiving
} else if (activeArchiveTask?.type === WorkflowTaskType.restore_asset) {
restoredStatus = AssetStatus.restoring
} else if (activeTask) {
restoredStatus = AssetStatus.processing
} else if (a.type === AssetType.file && a.storageKey?.isArchived) {
restoredStatus = AssetStatus.archived
}
await tx.asset.update({
where: { id: a.id },
data: {
isDeleted: false,
status: activeTask ? AssetStatus.processing : AssetStatus.processed,
status: restoredStatus,
deletedAt: null,
},
})
@@ -1805,7 +1959,12 @@ export class AssetService {
const attachmentAsset = await tx.asset.findUnique({
where: { id: attachmentId },
})
if (attachmentAsset) {
if (
attachmentAsset &&
(attachmentAsset.type === AssetType.attachment ||
attachmentAsset.type === AssetType.file) &&
attachmentAsset.projectId === a.projectId
) {
await tx.assetCommentAttachment.create({
data: {
assetId: attachmentAsset.id,
@@ -2352,6 +2511,7 @@ export class AssetService {
current: i === 0,
id: v.id,
name: v.name,
status: v.status,
previewUrl: preview?.thumbnailUrl || null,
createdAt: v.createdAt.toISOString(),
creator: v.creator
@@ -2445,7 +2605,15 @@ export class AssetService {
}
}
let media = latestVersion.media as PrismaJson.MediaInfo | null
const isOriginalUnavailable =
latestVersion.status === AssetStatus.archiving ||
latestVersion.status === AssetStatus.archived ||
latestVersion.status === AssetStatus.restoring ||
Boolean(latestVersion.storageKey?.isArchived)
let media = latestVersion.media
? (structuredClone(latestVersion.media) as PrismaJson.MediaInfo)
: null
if (media && media.videoPreview?.key) {
media.videoPreview.url = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
@@ -2454,11 +2622,15 @@ export class AssetService {
)
}
if (media && media.pdfTranscode?.key) {
media.pdfTranscode.url = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
media.pdfTranscode.key,
'GET',
)
if (isOriginalUnavailable && media.pdfTranscode.key === latestVersion.storageKey?.key) {
delete media.pdfTranscode.url
} else {
media.pdfTranscode.url = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
media.pdfTranscode.key,
'GET',
)
}
}
if (media && media.isHls && media.hls?.key) {
media.hls.url = `/api/files/${latestVersion.id}/m3u8/master.m3u8`
@@ -2523,6 +2695,7 @@ export class AssetService {
id: string
version: number
name: string
status: AssetStatus
previewUrl: string | null
createdAt: string
creator: { id: string; name: string | null; image?: string | null } | null
@@ -2546,6 +2719,7 @@ export class AssetService {
id: v.id,
version: versions.length - i,
name: v.name,
status: v.status,
previewUrl: preview?.thumbnailUrl || null,
createdAt: v.createdAt.toISOString(),
creator: v.creator
@@ -2605,8 +2779,11 @@ export class AssetService {
return Array.from(projectIds)
}
async getDownloadLinks(ids: string[]): Promise<Array<{ id: string; name: string; url: string }>> {
if (ids.length === 0) return []
async getDownloadLinks(ids: string[]): Promise<{
files: Array<{ id: string; name: string; url: string }>
skippedArchivedCount: number
}> {
if (ids.length === 0) return { files: [], skippedArchivedCount: 0 }
// Resolve starting IDs (dereference top-level symlinks)
const assets = await this.prismaClient.asset.findMany({
@@ -2625,13 +2802,14 @@ export class AssetService {
return asset.id
})
if (startingIds.length === 0) return []
if (startingIds.length === 0) return { files: [], skippedArchivedCount: 0 }
const descendants = await this.prismaClient.$queryRaw<
Array<{
id: string
name: string
type: string
status: string
parentId: string | null
storageKeyId: string | null
sortIndex: string | null
@@ -2639,16 +2817,16 @@ export class AssetService {
}>
>`
WITH RECURSIVE descendant AS (
SELECT id, name, type, parent_id, storage_key_id, sort_index, target_id, is_deleted
SELECT id, name, type, status::text AS status, parent_id, storage_key_id, sort_index, target_id, is_deleted
FROM assets
WHERE id IN (${Prisma.join(startingIds)}) AND is_deleted = false
UNION ALL
SELECT a.id, a.name, a.type, a.parent_id, a.storage_key_id, a.sort_index, a.target_id, a.is_deleted
SELECT a.id, a.name, a.type, a.status::text AS status, a.parent_id, a.storage_key_id, a.sort_index, a.target_id, a.is_deleted
FROM assets a
INNER JOIN descendant d ON a.parent_id = d.id
WHERE a.is_deleted = false
)
SELECT id, name, type, parent_id AS "parentId", storage_key_id AS "storageKeyId", sort_index AS "sortIndex", target_id AS "targetId" FROM descendant;
SELECT id, name, type, status, parent_id AS "parentId", storage_key_id AS "storageKeyId", sort_index AS "sortIndex", target_id AS "targetId" FROM descendant;
`
// Find any symlink in the descendants and resolve its target.
@@ -2657,6 +2835,7 @@ export class AssetService {
id: string
name: string
type: string
status: string
parentId: string | null
storageKeyId: string | null
sortIndex: string | null
@@ -2673,6 +2852,7 @@ export class AssetService {
id: t.id,
name: t.name,
type: t.type,
status: t.status,
parentId: t.parentId,
storageKeyId: t.storageKeyId,
sortIndex: t.sortIndex,
@@ -2737,16 +2917,29 @@ export class AssetService {
const storageKeys = await this.prismaClient.storageKey.findMany({
where: { id: { in: storageKeyIds } },
})
const storageKeyMap = new Map(storageKeys.map((k) => [k.id, k.key]))
const storageKeyMap = new Map(storageKeys.map((k) => [k.id, k]))
let skippedArchivedCount = 0
const downloadLinks = await Promise.all(
deduplicatedFiles.map(async (file) => {
const key = file.storageKeyId ? storageKeyMap.get(file.storageKeyId) : null
if (!key) return null
const sk = file.storageKeyId ? storageKeyMap.get(file.storageKeyId) : null
if (!sk?.key) return null
const isArchivedOrTransitioning =
file.status === AssetStatus.archiving ||
file.status === AssetStatus.archived ||
file.status === AssetStatus.restoring ||
sk.isArchived
if (isArchivedOrTransitioning) {
skippedArchivedCount++
return null
}
const url = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
key,
sk.key,
'GET',
true,
file.name,
@@ -2760,7 +2953,79 @@ export class AssetService {
}),
)
return downloadLinks.filter((link): link is { id: string; name: string; url: string } => !!link)
const filteredLinks = downloadLinks.filter(
(link): link is { id: string; name: string; url: string } => !!link,
)
return {
files: filteredLinks,
skippedArchivedCount,
}
}
async getStoragePaths(ids: string[]): Promise<GetStoragePathsResponse> {
if (ids.length === 0) return { paths: [] }
const targetAssetIds = await this.resolveLatestVersionIds(ids)
const uniqueTargetIds = Array.from(new Set(targetAssetIds))
const assets = await this.prismaClient.asset.findMany({
where: {
id: { in: uniqueTargetIds },
isDeleted: false,
type: {
notIn: [AssetType.folder, AssetType.root, AssetType.share_root, AssetType.share],
},
},
select: {
id: true,
name: true,
status: true,
storageKey: {
select: {
key: true,
isArchived: true,
},
},
},
})
const defaultBackend = getStorageBackend()
const mainBucket = process.env.S3_BUCKET || 'shumai'
const paths: StoragePathItem[] = []
for (const asset of assets) {
const sk = asset.storageKey
if (!sk?.key) continue
const isArchived =
asset.status === AssetStatus.archived ||
asset.status === AssetStatus.archiving ||
asset.status === AssetStatus.restoring ||
sk.isArchived
const archivePathInfo = isArchived ? getArchiveStoragePath(sk.key) : null
if (archivePathInfo) {
paths.push({
id: asset.id,
name: asset.name,
path: archivePathInfo.path,
backend: archivePathInfo.backend,
isArchived: true,
})
} else {
const storagePath = s3Service.getStoragePath(mainBucket, sk.key)
paths.push({
id: asset.id,
name: asset.name,
path: storagePath,
backend: defaultBackend,
isArchived,
})
}
}
return { paths }
}
/**
@@ -2772,7 +3037,13 @@ export class AssetService {
const targetAssetId = await this.resolveLatestVersionId(assetId)
const asset = await this.prismaClient.asset.findUnique({
where: { id: targetAssetId },
select: { id: true, name: true, parentId: true, storageKey: { select: { key: true } } },
select: {
id: true,
name: true,
status: true,
parentId: true,
storageKey: { select: { key: true, isArchived: true } },
},
})
if (!asset?.storageKey?.key) {
@@ -2786,6 +3057,8 @@ export class AssetService {
let storageKey = asset.storageKey.key
let assetDir = storageKey.substring(0, storageKey.lastIndexOf('/') + 1)
let targetName = asset.name
let targetStatus = asset.status
let targetIsArchived = asset.storageKey.isArchived
if (!key.startsWith(assetDir) && asset.parentId) {
// Check if requested key belongs to another version in the same version stack
@@ -2795,11 +3068,17 @@ export class AssetService {
isDeleted: false,
storageKey: { key: { startsWith: key.substring(0, key.lastIndexOf('/') + 1) } },
},
select: { name: true, storageKey: { select: { key: true } } },
select: {
name: true,
status: true,
storageKey: { select: { key: true, isArchived: true } },
},
})
if (sibling?.storageKey?.key) {
storageKey = sibling.storageKey.key
assetDir = storageKey.substring(0, storageKey.lastIndexOf('/') + 1)
targetStatus = sibling.status
targetIsArchived = sibling.storageKey.isArchived
if (sibling.name) {
targetName = sibling.name
}
@@ -2810,6 +3089,19 @@ export class AssetService {
throw new Error('Key does not belong to this asset')
}
if (key === storageKey) {
if (
targetStatus === AssetStatus.archiving ||
targetStatus === AssetStatus.archived ||
targetStatus === AssetStatus.restoring ||
targetIsArchived
) {
throw new HTTPException(409, {
message: 'Original file is archived. Restore the asset to download the original file.',
})
}
}
let downloadFilename: string | undefined
if (key === storageKey) {
downloadFilename = targetName ?? undefined
@@ -2841,6 +3133,24 @@ export class AssetService {
return s3Service.presign(process.env.S3_BUCKET || 'shumai', key, 'GET', true, downloadFilename)
}
async archiveAssets(
assetIds: string[],
options?: {
suppressBusyErrors?: boolean
autoArchivePolicy?: {
method: 'not_updated' | 'after_created'
cutoffDate: Date
restoreCutoffDate: Date
}
},
): Promise<ArchiveAssetsResponse> {
return assetArchiveService.archiveAssets(assetIds, options)
}
async restoreArchivedAssets(assetIds: string[]): Promise<RestoreArchivedAssetsResponse> {
return assetArchiveService.restoreArchivedAssets(assetIds)
}
async toPreviewInfo(
asset: Asset | AssetWithIncludes,
previewFormat?: PreviewFormat,
@@ -0,0 +1,464 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as fs from 'fs'
import * as path from 'path'
import {
AssetStatus,
prisma,
Prisma,
WorkflowTaskType,
WorkflowTaskStatus,
type User,
} from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { projectService } from '@shumai/core/src/project/project'
import { logger } from '@shumai/core/src/logger'
import { assetService } from './asset'
import { assetArchiveService } from './asset-archive'
import { resetArchiveStorageServiceForTests } from '../s3/archive-storage'
import {
AutoArchiveJobService,
autoArchiveJobService,
getAutoArchiveIntervalMs,
} from './auto-archive-job'
describe('AutoArchiveJobService', () => {
setupTestDbHooks()
const archiveRoot = path.join(process.cwd(), 'data-test-auto-archive')
let originalEnv: NodeJS.ProcessEnv
let owner: User
let teamId: string
let projectId: string
let rootFolderId: string
beforeEach(async () => {
originalEnv = { ...process.env }
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
resetArchiveStorageServiceForTests()
if (fs.existsSync(archiveRoot)) {
fs.rmSync(archiveRoot, { recursive: true, force: true })
}
owner = await prisma.user.create({
data: { name: 'Auto Archive Owner', email: 'auto-archive-owner@test.com', password: 'pw' },
})
const team = await prisma.team.create({
data: {
name: 'Auto Archive Team',
settings: {
autoArchive: {
enabled: true,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
},
},
})
teamId = team.id
await prisma.teamMember.create({ data: { teamId, userId: owner.id, role: 'owner' } })
const project = await projectService.createProject(owner, { teamId, name: 'Archive Project' })
projectId = project.id
rootFolderId = project.rootFolder!
})
afterEach(() => {
process.env = originalEnv
resetArchiveStorageServiceForTests()
if (fs.existsSync(archiveRoot)) {
fs.rmSync(archiveRoot, { recursive: true, force: true })
}
autoArchiveJobService.stop()
})
async function createTestFile(params: {
name: string
key: string
createdAt?: Date
updatedAt?: Date
status?: AssetStatus
}) {
const asset = await assetService.createAsset({
name: params.name,
type: 'file',
parentId: rootFolderId,
projectId,
key: params.key,
sizeByte: 2048,
contentType: 'video/mp4',
creatorId: owner.id,
})
const updated = await prisma.asset.update({
where: { id: asset.id },
data: {
status: params.status ?? AssetStatus.processed,
createdAt: params.createdAt ?? new Date(),
updatedAt: params.updatedAt ?? new Date(),
},
include: { storageKey: true },
})
return updated
}
describe('getAutoArchiveIntervalMs', () => {
it('returns default 1 hour (3600000ms) when env not specified', () => {
delete process.env.AUTO_ARCHIVE_INTERVAL_MINUTES
expect(getAutoArchiveIntervalMs()).toBe(3600000)
})
it('parses integer minute strings', () => {
process.env.AUTO_ARCHIVE_INTERVAL_MINUTES = '30'
expect(getAutoArchiveIntervalMs()).toBe(1800000)
})
it('parses decimal floating point minute strings', () => {
process.env.AUTO_ARCHIVE_INTERVAL_MINUTES = '0.02'
expect(getAutoArchiveIntervalMs()).toBe(1200)
})
})
it('skips scan when archive storage is not configured', async () => {
delete process.env.ARCHIVE_STORAGE_BACKEND
resetArchiveStorageServiceForTests()
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(0)
})
it('skips teams where autoArchive is disabled', async () => {
await prisma.team.update({
where: { id: teamId },
data: {
settings: {
autoArchive: {
enabled: false,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
},
},
})
const oldDate = new Date(Date.now() - 40 * 86_400_000)
await createTestFile({
name: 'old-file.mp4',
key: 'files/test/old-file.mp4',
updatedAt: oldDate,
})
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(0)
})
it('archives assets exceeding not_updated threshold while leaving recent assets untouched', async () => {
const oldDate = new Date(Date.now() - 35 * 86_400_000)
const recentDate = new Date(Date.now() - 5 * 86_400_000)
const oldFile = await createTestFile({
name: 'old-file.mp4',
key: 'files/test/old-file.mp4',
updatedAt: oldDate,
})
const recentFile = await createTestFile({
name: 'recent-file.mp4',
key: 'files/test/recent-file.mp4',
updatedAt: recentDate,
})
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(1)
const refreshedOld = await prisma.asset.findUniqueOrThrow({ where: { id: oldFile.id } })
const refreshedRecent = await prisma.asset.findUniqueOrThrow({ where: { id: recentFile.id } })
expect(refreshedOld.status).toBe(AssetStatus.archiving)
expect(refreshedRecent.status).toBe(AssetStatus.processed)
const task = await prisma.workflowTask.findFirst({
where: { assetId: oldFile.id, type: WorkflowTaskType.archive_asset },
})
expect(task).not.toBeNull()
expect(task?.status).toBe(WorkflowTaskStatus.pending)
})
it('archives assets exceeding after_created threshold and respects postRestoreGraceDays', async () => {
await prisma.team.update({
where: { id: teamId },
data: {
settings: {
autoArchive: {
enabled: true,
method: 'after_created',
days: 30,
postRestoreGraceDays: 14,
},
},
},
})
const oldCreatedDate = new Date(Date.now() - 45 * 86_400_000)
const recentlyRestoredDate = new Date(Date.now() - 2 * 86_400_000)
const oldRestoredDate = new Date(Date.now() - 20 * 86_400_000)
// Old creation, but restored 2 days ago (within 14-day grace period) -> should NOT archive
const restoredRecently = await createTestFile({
name: 'restored-recently.mp4',
key: 'files/test/restored-recently.mp4',
createdAt: oldCreatedDate,
updatedAt: recentlyRestoredDate,
})
// Old creation, restored 20 days ago (> 14-day grace period) -> SHOULD archive
const restoredLongAgo = await createTestFile({
name: 'restored-long-ago.mp4',
key: 'files/test/restored-long-ago.mp4',
createdAt: oldCreatedDate,
updatedAt: oldRestoredDate,
})
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(1)
const refreshedRecently = await prisma.asset.findUniqueOrThrow({
where: { id: restoredRecently.id },
})
const refreshedLongAgo = await prisma.asset.findUniqueOrThrow({
where: { id: restoredLongAgo.id },
})
expect(refreshedRecently.status).toBe(AssetStatus.processed)
expect(refreshedLongAgo.status).toBe(AssetStatus.archiving)
})
it('skips storage key if any sibling file sharing the key was recently modified', async () => {
const oldDate = new Date(Date.now() - 40 * 86_400_000)
const recentDate = new Date()
const sharedKey = 'files/test/shared-video.mp4'
const primaryFile = await createTestFile({
name: 'primary.mp4',
key: sharedKey,
updatedAt: oldDate,
})
// Create a sibling copy sharing the same storageKeyId but modified recently
const siblingCopy = await assetService.createAsset({
name: 'sibling-copy.mp4',
type: 'file',
parentId: rootFolderId,
projectId,
key: sharedKey,
sizeByte: 2048,
contentType: 'video/mp4',
creatorId: owner.id,
})
await prisma.asset.update({
where: { id: siblingCopy.id },
data: {
status: AssetStatus.processed,
updatedAt: recentDate,
},
})
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(0)
const refreshedPrimary = await prisma.asset.findUniqueOrThrow({
where: { id: primaryFile.id },
})
expect(refreshedPrimary.status).toBe(AssetStatus.processed)
})
it('handles race conditions: concurrent archive attempts cleanly serialize without duplicate tasks', async () => {
const oldDate = new Date(Date.now() - 50 * 86_400_000)
const file = await createTestFile({
name: 'concurrent-test.mp4',
key: 'files/test/concurrent-test.mp4',
updatedAt: oldDate,
})
const job = new AutoArchiveJobService()
// Trigger auto-archive and manual archive concurrently
const [jobRes, manualRes] = await Promise.all([
job.processAutoArchive(),
assetService.archiveAssets([file.id], { suppressBusyErrors: true }),
])
// Together they should archive the asset exactly once
const totalArchived = jobRes.archivedCount + manualRes.archivedAssetIds.length
expect(totalArchived).toBe(1)
// Verify exactly ONE WorkflowTask was created
const tasks = await prisma.workflowTask.findMany({
where: { assetId: file.id, type: WorkflowTaskType.archive_asset },
})
expect(tasks).toHaveLength(1)
const refreshedFile = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
expect(refreshedFile.status).toBe(AssetStatus.archiving)
})
it('prevents candidate starvation: storage keys with stuck siblings are ignored so eligible candidates are not blocked', async () => {
const oldDate = new Date(Date.now() - 40 * 86_400_000)
// Key 1: has an old file AND a stuck sibling with status 'failed'
const stuckKey = 'files/test/stuck-sibling.mp4'
const fileWithStuckSibling = await createTestFile({
name: 'stuck-primary.mp4',
key: stuckKey,
updatedAt: oldDate,
})
const stuckSibling = await assetService.createAsset({
name: 'stuck-sibling-failed.mp4',
type: 'file',
parentId: rootFolderId,
projectId,
key: stuckKey,
sizeByte: 2048,
contentType: 'video/mp4',
creatorId: owner.id,
})
await prisma.asset.update({
where: { id: stuckSibling.id },
data: {
status: AssetStatus.failed,
updatedAt: oldDate,
},
})
// Key 2: fully eligible old file
const eligibleFile = await createTestFile({
name: 'eligible-siblingless.mp4',
key: 'files/test/eligible-siblingless.mp4',
updatedAt: oldDate,
})
const job = new AutoArchiveJobService()
// With batchSize 1, the query must pick the eligible file instead of getting stuck on Key 1
const res = await job.processAutoArchive(1)
expect(res.archivedCount).toBe(1)
const refreshedEligible = await prisma.asset.findUniqueOrThrow({
where: { id: eligibleFile.id },
})
expect(refreshedEligible.status).toBe(AssetStatus.archiving)
const refreshedStuck = await prisma.asset.findUniqueOrThrow({
where: { id: fileWithStuckSibling.id },
})
expect(refreshedStuck.status).toBe(AssetStatus.processed)
})
it('skips scan when another process holds the transaction advisory lock', async () => {
const queryRawSpy = vi.spyOn(prisma, '$queryRaw').mockImplementation((query: unknown) => {
const sqlText = String((query as { strings?: string[] })?.strings || '')
if (sqlText.includes('pg_try_advisory_xact_lock')) {
return Promise.resolve([{ locked: false }]) as unknown as Prisma.PrismaPromise<unknown>
}
queryRawSpy.mockRestore()
return prisma.$queryRaw(query as TemplateStringsArray)
})
try {
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(0)
} finally {
queryRawSpy.mockRestore()
}
})
it('logs info level message on job start and finish with totalArchived count', async () => {
const infoSpy = vi.spyOn(logger, 'info')
const oldDate = new Date(Date.now() - 40 * 86_400_000)
await createTestFile({
name: 'log-test.mp4',
key: 'files/test/log-test.mp4',
updatedAt: oldDate,
})
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(1)
expect(infoSpy).toHaveBeenCalledWith('Auto-archive job started')
expect(infoSpy).toHaveBeenCalledWith({ totalArchived: 1 }, 'Auto-archive job finished')
infoSpy.mockRestore()
})
it('does not log info level messages when no files are eligible to archive', async () => {
const infoSpy = vi.spyOn(logger, 'info')
const job = new AutoArchiveJobService()
const res = await job.processAutoArchive()
expect(res.archivedCount).toBe(0)
expect(infoSpy).not.toHaveBeenCalledWith('Auto-archive job started')
expect(infoSpy).not.toHaveBeenCalledWith(expect.anything(), 'Auto-archive job finished')
infoSpy.mockRestore()
})
it('does not archive non-file assets even if their status is processed and age criteria met', async () => {
const oldDate = new Date(Date.now() - 50 * 86_400_000)
const folder = await assetService.createAsset({
name: 'Old Folder',
type: 'folder',
projectId,
parentId: rootFolderId,
creatorId: owner.id,
})
await prisma.asset.update({
where: { id: folder.id },
data: {
status: AssetStatus.processed,
updatedAt: oldDate,
createdAt: oldDate,
},
})
const res = await assetArchiveService.archiveAssets([folder.id], {
suppressBusyErrors: true,
autoArchivePolicy: {
method: 'not_updated',
cutoffDate: new Date(),
restoreCutoffDate: new Date(),
},
})
expect(res.archivedAssetIds).toHaveLength(0)
const refreshedFolder = await prisma.asset.findUniqueOrThrow({
where: { id: folder.id },
})
expect(refreshedFolder.status).toBe(AssetStatus.processed)
})
it('can start and stop without errors', () => {
const job = new AutoArchiveJobService()
job.start(1000, 5000)
job.stop()
})
})
+197
View File
@@ -0,0 +1,197 @@
import { prisma, Prisma } from '@shumai/db'
import { isArchiveStorageEnabled } from '@shumai/core/src/s3/archive-storage'
import { assetArchiveService } from '@shumai/core/src/asset/asset-archive'
import { logger } from '@shumai/core/src/logger'
export function getAutoArchiveIntervalMs(): number {
const raw = process.env.AUTO_ARCHIVE_INTERVAL_MINUTES
if (raw) {
const parsed = Number.parseFloat(raw)
if (!Number.isNaN(parsed) && parsed > 0) {
return Math.round(parsed * 60 * 1000)
}
}
return 60 * 60 * 1000 // default 1 hour
}
interface TeamArchiveBatch {
teamId: string
assetIds: string[]
policy: {
method: 'not_updated' | 'after_created'
cutoffDate: Date
restoreCutoffDate: Date
}
}
export class AutoArchiveJobService {
private isRunning = false
private isProcessing = false
private timer: NodeJS.Timeout | null = null
start(initialDelayMs = 60_000, intervalMs?: number): void {
if (this.isRunning) return
this.isRunning = true
logger.info('Starting auto-archive background job')
const run = async () => {
if (!this.isRunning) return
try {
await this.processAutoArchive()
} catch (err: unknown) {
logger.error({ err }, 'Error in auto-archive background job')
}
if (this.isRunning) {
const nextInterval = intervalMs ?? getAutoArchiveIntervalMs()
this.timer = setTimeout(run, nextInterval)
}
}
this.timer = setTimeout(run, initialDelayMs)
}
stop(): void {
this.isRunning = false
if (this.timer) {
clearTimeout(this.timer)
this.timer = null
}
logger.info('Stopped auto-archive background job')
}
async processAutoArchive(batchSize = 100): Promise<{ archivedCount: number }> {
if (!isArchiveStorageEnabled()) {
logger.debug('Archive storage is not configured; skipping auto-archive run')
return { archivedCount: 0 }
}
if (this.isProcessing) {
logger.debug('Auto-archive job turn already in progress, skipping')
return { archivedCount: 0 }
}
this.isProcessing = true
try {
let totalArchived = 0
// Phase 1: Query & Scan under transaction-scoped advisory lock.
// Automatically released as soon as this transaction completes.
const teamBatches = await prisma.$transaction(async (tx) => {
const lockRes = await tx.$queryRaw<Array<{ locked: boolean }>>`
SELECT pg_try_advisory_xact_lock(hashtext('auto_archive_job')) AS locked
`
if (!lockRes || !lockRes[0]?.locked) {
logger.debug(
'Could not acquire transaction advisory lock for auto-archive scan, skipping',
)
return null
}
// Query only teams where autoArchive is enabled via JSON filter
const eligibleTeams = await tx.team.findMany({
where: {
settings: {
path: ['autoArchive', 'enabled'],
equals: true,
},
},
select: { id: true, settings: true },
})
if (eligibleTeams.length === 0) {
return []
}
const batches: TeamArchiveBatch[] = []
for (const team of eligibleTeams) {
const settings = (team.settings || {}) as PrismaJson.Settings
const autoArchive = settings.autoArchive
const method = autoArchive?.method || 'not_updated'
const days = Math.max(1, autoArchive?.days || 30)
const postRestoreGraceDays = Math.max(1, autoArchive?.postRestoreGraceDays || 30)
const now = Date.now()
const cutoffDate = new Date(now - days * 86_400_000)
const restoreCutoffDate = new Date(now - postRestoreGraceDays * 86_400_000)
// Query candidate assets whose storage keys have NO ineligible sibling.
// NOT EXISTS eliminates starvation from stuck/recent siblings at the database level.
const candidateRows = await tx.$queryRaw<Array<{ id: string }>>`
SELECT a.id
FROM assets a
JOIN projects p ON a.project_id = p.id
JOIN storage_keys sk ON a.storage_key_id = sk.id
WHERE p.team_id = ${team.id}
AND a.type = 'file'::"AssetType"
AND a.status = 'processed'::"AssetStatus"
AND a.is_deleted = false
AND sk.is_archived = false
AND ${
method === 'not_updated'
? Prisma.sql`a.updated_at <= ${cutoffDate}`
: Prisma.sql`a.created_at <= ${cutoffDate} AND a.updated_at <= ${restoreCutoffDate}`
}
AND NOT EXISTS (
SELECT 1
FROM assets sib
WHERE sib.storage_key_id = a.storage_key_id
AND sib.is_deleted = false
AND (
sib.type != 'file'::"AssetType"
OR sib.status != 'processed'::"AssetStatus"
OR ${
method === 'not_updated'
? Prisma.sql`sib.updated_at > ${cutoffDate}`
: Prisma.sql`sib.created_at > ${cutoffDate} OR sib.updated_at > ${restoreCutoffDate}`
}
)
)
ORDER BY a.id ASC
LIMIT ${batchSize};
`
if (candidateRows.length > 0) {
batches.push({
teamId: team.id,
assetIds: candidateRows.map((r) => r.id),
policy: { method, cutoffDate, restoreCutoffDate },
})
}
}
return batches
})
if (!teamBatches || teamBatches.length === 0) {
return { archivedCount: 0 }
}
logger.info('Auto-archive job started')
// Phase 2: Execute archival in chunks per team batch.
// Each asset/storageKey transition is protected by row-level lock and re-verifies policy.
const chunkSize = 50
for (const batch of teamBatches) {
for (let i = 0; i < batch.assetIds.length; i += chunkSize) {
const chunk = batch.assetIds.slice(i, i + chunkSize)
const result = await assetArchiveService.archiveAssets(chunk, {
suppressBusyErrors: true,
autoArchivePolicy: batch.policy,
})
totalArchived += result.archivedAssetIds.length
}
}
logger.info({ totalArchived }, 'Auto-archive job finished')
return { archivedCount: totalArchived }
} finally {
this.isProcessing = false
}
}
}
export const autoArchiveJobService = new AutoArchiveJobService()
+46 -1
View File
@@ -1,7 +1,7 @@
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { auth, authService } from './auth'
import { auth, authService, isLoopbackHost, isMcpResourceValid } from './auth'
import { userMetadataService } from '../user-metadata/user-metadata'
import { teamService } from '@shumai/core/src/team/team'
@@ -326,3 +326,48 @@ describe('AuthService Password Reset', () => {
}
})
})
describe('MCP Resource Validation & Loopback Checking', () => {
it('correctly identifies loopback hostnames', () => {
expect(isLoopbackHost('localhost')).toBe(true)
expect(isLoopbackHost('127.0.0.1')).toBe(true)
expect(isLoopbackHost('127.0.0.254')).toBe(true)
expect(isLoopbackHost('[::1]')).toBe(true)
expect(isLoopbackHost('43.133.100.178')).toBe(false)
expect(isLoopbackHost('192.168.1.100')).toBe(false)
expect(isLoopbackHost('10.0.0.1')).toBe(false)
expect(isLoopbackHost('example.com')).toBe(false)
expect(isLoopbackHost('128.0.0.1')).toBe(false)
})
it('validates MCP resource URLs according to RFC 9728 and MCP security rules', () => {
// Loopback HTTP URLs are allowed for development
expect(isMcpResourceValid('http://localhost:3000/mcp')).toEqual({ valid: true })
expect(isMcpResourceValid('http://127.0.0.1:3000/mcp')).toEqual({ valid: true })
expect(isMcpResourceValid('http://[::1]:3000/mcp')).toEqual({ valid: true })
// HTTPS is allowed for any host (including domains and remote IPs)
expect(isMcpResourceValid('https://shumai.example.com/mcp')).toEqual({ valid: true })
expect(isMcpResourceValid('https://43.133.100.178:3000/mcp')).toEqual({ valid: true })
// Remote HTTP IPs and hostnames are rejected (the user issue)
const remoteIpResult = isMcpResourceValid('http://43.133.100.178:3000/mcp')
expect(remoteIpResult.valid).toBe(false)
expect(remoteIpResult.error).toContain(
'MCP resource URL must use HTTPS, except for localhost or loopback IP development URLs',
)
const remoteDomainResult = isMcpResourceValid('http://shumai.example.com/mcp')
expect(remoteDomainResult.valid).toBe(false)
expect(remoteDomainResult.error).toContain(
'MCP resource URL must use HTTPS, except for localhost or loopback IP development URLs',
)
// Invalid URL formats
expect(isMcpResourceValid('not-a-url').valid).toBe(false)
expect(isMcpResourceValid('http://user:pass@localhost:3000/mcp').valid).toBe(false)
expect(isMcpResourceValid('http://localhost:3000/mcp#frag').valid).toBe(false)
expect(isMcpResourceValid('http://localhost:3000/mcp?query=1').valid).toBe(false)
})
})
+69 -10
View File
@@ -8,6 +8,7 @@ import { teamService } from '@shumai/core/src/team/team'
import { inviteService } from '@shumai/core/src/invite/invite'
import { createAuthMiddleware } from 'better-auth/api'
import { userMetadataService } from '@shumai/core/src/user-metadata/user-metadata'
import { logger } from '@shumai/core/src/logger'
interface GeneratedResetLink {
url: string
@@ -22,22 +23,80 @@ export const authBaseUrl = (
export const mcpResource = `${authBaseUrl}/mcp`
export { hashPassword } from 'better-auth/crypto'
/**
* Checks whether a host is a loopback address (localhost, 127.0.0.0/8, or [::1]).
*/
export function isLoopbackHost(hostname: string): boolean {
const ipv4Octets = hostname.split('.')
const isIpv4Loopback =
ipv4Octets.length === 4 &&
ipv4Octets[0] === '127' &&
ipv4Octets.every((octet) => /^\d+$/.test(octet) && Number(octet) <= 255)
return hostname === 'localhost' || hostname === '[::1]' || isIpv4Loopback
}
/**
* Validates whether an MCP resource URL meets RFC 9728 and MCP security requirements
* (HTTPS required unless using a localhost or loopback IP address).
*/
export function isMcpResourceValid(resource: string): { valid: boolean; error?: string } {
if (typeof resource !== 'string' || !resource) {
return { valid: false, error: 'MCP resource must be a non-empty string URL' }
}
let url: URL
try {
url = new URL(resource)
} catch {
return { valid: false, error: 'MCP resource must be an absolute URL' }
}
if (url.username || url.password) {
return { valid: false, error: 'MCP resource URL must not contain credentials' }
}
if (resource.includes('#')) {
return { valid: false, error: 'MCP resource URL must not contain a fragment' }
}
if (resource.includes('?')) {
return { valid: false, error: 'MCP resource URL must not contain a query' }
}
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && isLoopbackHost(url.hostname))) {
return {
valid: false,
error:
'MCP resource URL must use HTTPS, except for localhost or loopback IP development URLs',
}
}
return { valid: true }
}
const mcpValidation = isMcpResourceValid(mcpResource)
export const isMcpOauthEnabled = mcpValidation.valid
if (!isMcpOauthEnabled) {
logger.error(
{ mcpResource, error: mcpValidation.error },
'MCP OAuth 2.1 is disabled: resource URL must use HTTPS for remote hosts. The main server and Web UI will continue running normally, but MCP OAuth clients cannot connect until HTTPS is configured.',
)
}
export const auth = betterAuth({
database: prismaAdapter(prisma, {
provider: 'postgresql',
}),
plugins: [
jwt(),
mcp({
loginPage: '/login',
consentPage: '/consent',
resource: mcpResource,
resources: [{ identifier: mcpResource, allowedScopes: ['mcp'] }],
scopes: ['openid', 'profile', 'email', 'offline_access', 'mcp'],
allowDynamicClientRegistration: true,
allowUnauthenticatedClientRegistration: true,
clientRegistrationDefaultApplicationType: 'infer',
}),
...(isMcpOauthEnabled
? [
mcp({
loginPage: '/login',
consentPage: '/consent',
resource: mcpResource,
resources: [{ identifier: mcpResource, allowedScopes: ['mcp'] }],
scopes: ['openid', 'profile', 'email', 'offline_access', 'mcp'],
allowDynamicClientRegistration: true,
allowUnauthenticatedClientRegistration: true,
clientRegistrationDefaultApplicationType: 'infer',
}),
]
: []),
],
emailAndPassword: {
enabled: true,
@@ -0,0 +1,38 @@
import { describe, expect, it } from 'vitest'
import { createGuestToken, verifyGuestToken } from './guest-token'
describe('guest-token', () => {
it('creates and verifies a valid token for the expected shareId', () => {
const token = createGuestToken({ userId: 'u1', shareId: 'share1' })
const verified = verifyGuestToken(token, 'share1')
expect(verified).not.toBeNull()
expect(verified?.userId).toBe('u1')
expect(verified?.shareId).toBe('share1')
})
it('rejects a token when verified against a different shareId', () => {
const token = createGuestToken({ userId: 'u1', shareId: 'share1' })
const verified = verifyGuestToken(token, 'share2')
expect(verified).toBeNull()
})
it('rejects an expired token', () => {
const token = createGuestToken({ userId: 'u1', shareId: 'share1' }, -10)
const verified = verifyGuestToken(token, 'share1')
expect(verified).toBeNull()
})
it('rejects a tampered signature', () => {
const token = createGuestToken({ userId: 'u1', shareId: 'share1' })
const [payload] = token.split('.')
const tampered = `${payload}.invalidsignature`
const verified = verifyGuestToken(tampered, 'share1')
expect(verified).toBeNull()
})
it('rejects malformed token strings', () => {
expect(verifyGuestToken('', 'share1')).toBeNull()
expect(verifyGuestToken('singlepart', 'share1')).toBeNull()
expect(verifyGuestToken('a.b.c', 'share1')).toBeNull()
})
})
+69
View File
@@ -0,0 +1,69 @@
import * as crypto from 'crypto'
export interface GuestTokenPayload {
userId: string
shareId: string
expiresAt: number
}
const GUEST_TOKEN_CONTEXT = 'shumai-guest-token:v1'
const DEFAULT_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days
function getSecret(): string {
return process.env.BETTER_AUTH_SECRET || 'shumai-guest-secret'
}
function signPayload(payloadB64: string): string {
return crypto
.createHmac('sha256', getSecret())
.update(`${GUEST_TOKEN_CONTEXT}:${payloadB64}`)
.digest('base64url')
}
export function createGuestToken(
params: { userId: string; shareId: string },
ttlSeconds: number = DEFAULT_TTL_SECONDS,
): string {
const payload: GuestTokenPayload = {
userId: params.userId,
shareId: params.shareId,
expiresAt: Date.now() + ttlSeconds * 1000,
}
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url')
const signature = signPayload(payloadB64)
return `${payloadB64}.${signature}`
}
export function verifyGuestToken(token: string, expectedShareId: string): GuestTokenPayload | null {
if (!token) return null
const [payloadB64, signature, ...rest] = token.split('.')
if (!payloadB64 || !signature || rest.length > 0) {
return null
}
const expectedSignature = signPayload(payloadB64)
const actualBuf = Buffer.from(signature)
const expectedBuf = Buffer.from(expectedSignature)
if (actualBuf.length !== expectedBuf.length || !crypto.timingSafeEqual(actualBuf, expectedBuf)) {
return null
}
try {
const payload = JSON.parse(
Buffer.from(payloadB64, 'base64url').toString('utf8'),
) as GuestTokenPayload
if (!payload.userId || !payload.shareId || typeof payload.expiresAt !== 'number') {
return null
}
if (payload.shareId !== expectedShareId) {
return null
}
if (Date.now() > payload.expiresAt) {
return null
}
return payload
} catch {
return null
}
}
+2
View File
@@ -1,5 +1,7 @@
export * from './agent/agent'
export * from './asset/asset'
export * from './asset/asset-archive'
export * from './asset/auto-archive-job'
export * from './auth/auth'
export * from './authz/authz'
export * from './collection/collection'
+29
View File
@@ -141,4 +141,33 @@ describe('Invite Service', () => {
expect(updatedTm?.scope).toBe('team')
expect(updatedTm?.role).toBe('editor')
})
it('atomically claims invite and rejects concurrent redemptions', async () => {
const inv = await inviteService.createTeamInvite({
teamId: team.id,
role: 'editor',
inviterId: admin.id,
})
const userA = await prisma.user.create({
data: { name: 'userA', email: 'userA-invite@example.com', password: 'pass' },
})
const userB = await prisma.user.create({
data: { name: 'userB', email: 'userB-invite@example.com', password: 'pass' },
})
const results = await Promise.allSettled([
inviteService.consumeInvite(inv.code, userA.id),
inviteService.consumeInvite(inv.code, userB.id),
])
const fulfilled = results.filter((r) => r.status === 'fulfilled')
const rejected = results.filter((r) => r.status === 'rejected')
expect(fulfilled.length).toBe(1)
expect(rejected.length).toBe(1)
if (rejected[0].status === 'rejected') {
expect(rejected[0].reason.message).toContain('invite code already used')
}
})
})
+5 -6
View File
@@ -84,7 +84,11 @@ export class InviteService {
include: { team: true, project: true },
})
if (inv.used) {
const claimed = await tx.invite.updateMany({
where: { id: inv.id, used: false },
data: { used: true },
})
if (claimed.count !== 1) {
throw new Error('invite code already used')
}
@@ -155,11 +159,6 @@ export class InviteService {
})
}
}
await tx.invite.update({
where: { id: inv.id },
data: { used: true },
})
})
}
}
@@ -0,0 +1,224 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import * as fs from 'fs'
import * as path from 'path'
import { AssetStatus, prisma, type User } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { AuditAction } from '@shumai/dtos'
import { projectService } from '@shumai/core/src/project/project'
import { assetService } from '@shumai/core/src/asset/asset'
import { resetArchiveStorageServiceForTests } from '@shumai/core/src/s3/archive-storage'
import {
archiveAssetsOp,
restoreArchivedAssetsOp,
runOperation,
type OperationContext,
} from './index'
describe('archive_assets and restore_archived_assets Operations', () => {
setupTestDbHooks()
const archiveRoot = path.join(process.cwd(), 'data-test-ops-archive')
let originalEnv: NodeJS.ProcessEnv
let owner: User
let editor: User
let reviewer: User
let outsider: User
let teamId: string
let projectId: string
let rootFolderId: string
const as = (user: User, via: OperationContext['via'] = 'cli'): OperationContext => ({
userId: user.id,
via,
})
beforeEach(async () => {
originalEnv = { ...process.env }
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
resetArchiveStorageServiceForTests()
owner = await prisma.user.create({
data: { name: 'Owner', email: 'arch-ops-owner@test.com', password: 'pw' },
})
editor = await prisma.user.create({
data: { name: 'Editor', email: 'arch-ops-editor@test.com', password: 'pw' },
})
reviewer = await prisma.user.create({
data: { name: 'Reviewer', email: 'arch-ops-reviewer@test.com', password: 'pw' },
})
outsider = await prisma.user.create({
data: { name: 'Outsider', email: 'arch-ops-outsider@test.com', password: 'pw' },
})
const team = await prisma.team.create({ data: { name: 'Arch Ops Team' } })
teamId = team.id
await prisma.teamMember.create({ data: { teamId, userId: owner.id, role: 'owner' } })
await prisma.teamMember.create({ data: { teamId, userId: editor.id, role: 'editor' } })
await prisma.teamMember.create({ data: { teamId, userId: reviewer.id, role: 'reviewer' } })
const project = await projectService.createProject(owner, { teamId, name: 'Arch Ops Project' })
projectId = project.id
rootFolderId = project.rootFolder!
})
afterEach(() => {
process.env = originalEnv
resetArchiveStorageServiceForTests()
if (fs.existsSync(archiveRoot)) {
fs.rmSync(archiveRoot, { recursive: true, force: true })
}
})
it('allows editor and owner to archive and restore assets and records audit entries', async () => {
const file = await assetService.createAsset({
name: 'take1.mov',
type: 'file',
parentId: rootFolderId,
projectId,
key: 'files/01OP/take1.mov',
sizeByte: 4096,
contentType: 'video/quicktime',
creatorId: owner.id,
})
await prisma.asset.update({
where: { id: file.id },
data: { status: AssetStatus.processed },
})
const archiveOut = await runOperation(archiveAssetsOp, as(editor, 'api'), {
assetIds: [file.id],
})
expect(archiveOut.archivedAssetIds).toEqual([file.id])
expect(archiveOut.taskIds).toHaveLength(1)
expect(archiveOut.skippedCount).toBe(0)
const archiveAudit = await prisma.auditLog.findMany({
where: { action: AuditAction.asset_archive },
})
expect(archiveAudit).toHaveLength(1)
expect(archiveAudit[0]).toMatchObject({
teamId,
projectId,
userId: editor.id,
itemId: file.id,
via: 'api',
})
// Transition to archived state to test restore_archived_assets
await prisma.storageKey.update({
where: { key: 'files/01OP/take1.mov' },
data: { isArchived: true },
})
await prisma.asset.update({
where: { id: file.id },
data: { status: AssetStatus.archived },
})
const restoreOut = await runOperation(restoreArchivedAssetsOp, as(owner, 'mcp'), {
assetIds: [file.id],
})
expect(restoreOut.restoredAssetIds).toEqual([file.id])
expect(restoreOut.taskIds).toHaveLength(1)
const restoreAudit = await prisma.auditLog.findMany({
where: { action: AuditAction.asset_restore_archive },
})
expect(restoreAudit).toHaveLength(1)
expect(restoreAudit[0]).toMatchObject({
teamId,
projectId,
userId: owner.id,
itemId: file.id,
via: 'mcp',
})
})
it('denies reviewer and outsider with 403 and causes no side effects or audit entries', async () => {
const file = await assetService.createAsset({
name: 'protected.mov',
type: 'file',
parentId: rootFolderId,
projectId,
key: 'files/01PROT/protected.mov',
sizeByte: 2048,
contentType: 'video/quicktime',
creatorId: owner.id,
})
await prisma.asset.update({
where: { id: file.id },
data: { status: AssetStatus.processed },
})
for (const user of [reviewer, outsider]) {
await expect(
runOperation(archiveAssetsOp, as(user), { assetIds: [file.id] }),
).rejects.toMatchObject({ status: 403 })
}
const row = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
expect(row.status).toBe(AssetStatus.processed)
expect(await prisma.auditLog.count({ where: { action: AuditAction.asset_archive } })).toBe(0)
await prisma.storageKey.update({
where: { key: 'files/01PROT/protected.mov' },
data: { isArchived: true },
})
await prisma.asset.update({
where: { id: file.id },
data: { status: AssetStatus.archived },
})
for (const user of [reviewer, outsider]) {
await expect(
runOperation(restoreArchivedAssetsOp, as(user), { assetIds: [file.id] }),
).rejects.toMatchObject({ status: 403 })
}
const archivedRow = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
expect(archivedRow.status).toBe(AssetStatus.archived)
expect(
await prisma.auditLog.count({ where: { action: AuditAction.asset_restore_archive } }),
).toBe(0)
})
it('rejects symlinks directly with 400 and causes no side effects', async () => {
const file = await assetService.createAsset({
name: 'real.mov',
type: 'file',
parentId: rootFolderId,
projectId,
key: 'files/01REAL/real.mov',
sizeByte: 2048,
contentType: 'video/quicktime',
creatorId: owner.id,
})
await prisma.asset.update({
where: { id: file.id },
data: { status: AssetStatus.processed },
})
const symlink = await prisma.asset.create({
data: {
name: 'symlink.mov',
type: 'symlink',
parentId: rootFolderId,
projectId,
targetId: file.id,
status: AssetStatus.processed,
creatorId: owner.id,
},
})
await expect(
runOperation(archiveAssetsOp, as(owner), { assetIds: [symlink.id] }),
).rejects.toMatchObject({ status: 400, message: 'Symlinks cannot be archived or restored' })
const unchangedFile = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
expect(unchangedFile.status).toBe(AssetStatus.processed)
expect(await prisma.auditLog.count({ where: { action: AuditAction.asset_archive } })).toBe(0)
await expect(
runOperation(restoreArchivedAssetsOp, as(owner), { assetIds: [symlink.id] }),
).rejects.toMatchObject({ status: 400, message: 'Symlinks cannot be archived or restored' })
})
})
@@ -0,0 +1,23 @@
import { archiveAssets } from '@shumai/operations'
import { AuditAction } from '@shumai/dtos'
import { assetService } from '@shumai/core/src/asset/asset'
import { Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { implementOperation } from '../implement'
export const archiveAssetsOp = implementOperation(archiveAssets, {
authz: (input) =>
input.assetIds.map((id) => ({
permission: Permission.Edit,
type: ResourceType.Asset,
id,
})),
execute: async (input) => {
return assetService.archiveAssets(input.assetIds)
},
audit: (input) =>
input.assetIds.map((assetId) => ({
action: AuditAction.asset_archive,
itemId: assetId,
contextAssetId: assetId,
})),
})
@@ -0,0 +1,23 @@
import { restoreArchivedAssets } from '@shumai/operations'
import { AuditAction } from '@shumai/dtos'
import { assetService } from '@shumai/core/src/asset/asset'
import { Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { implementOperation } from '../implement'
export const restoreArchivedAssetsOp = implementOperation(restoreArchivedAssets, {
authz: (input) =>
input.assetIds.map((id) => ({
permission: Permission.Edit,
type: ResourceType.Asset,
id,
})),
execute: async (input) => {
return assetService.restoreArchivedAssets(input.assetIds)
},
audit: (input) =>
input.assetIds.map((assetId) => ({
action: AuditAction.asset_restore_archive,
itemId: assetId,
contextAssetId: assetId,
})),
})
+2
View File
@@ -10,6 +10,7 @@ export {
} from './implement'
export { runOperation } from './run'
export { getOperation, getOperationsForSurface, operations } from './registry'
export { archiveAssetsOp } from './asset/archive-assets'
export { copyAssetsOp } from './asset/copy-assets'
export { createFolderOp } from './asset/create-folder'
export { deleteAssetOp } from './asset/delete-asset'
@@ -24,6 +25,7 @@ export {
waitBudgetMs,
} from './asset/read-asset'
export { renameAssetOp } from './asset/rename-asset'
export { restoreArchivedAssetsOp } from './asset/restore-archived-assets'
export { searchAssetsOp } from './asset/search-assets'
export { createCommentOp } from './comment/create-comment'
export { listCommentsOp } from './comment/list-comments'
+4
View File
@@ -1,4 +1,5 @@
import type { OperationName, Surface } from '@shumai/operations'
import { archiveAssetsOp } from './asset/archive-assets'
import { copyAssetsOp } from './asset/copy-assets'
import { createFolderOp } from './asset/create-folder'
import { deleteAssetOp } from './asset/delete-asset'
@@ -8,6 +9,7 @@ import { listAssetsOp } from './asset/list-assets'
import { moveAssetsOp } from './asset/move-assets'
import { getAssetRenderOp, readAssetOp } from './asset/read-asset'
import { renameAssetOp } from './asset/rename-asset'
import { restoreArchivedAssetsOp } from './asset/restore-archived-assets'
import { searchAssetsOp } from './asset/search-assets'
import { createCommentOp } from './comment/create-comment'
import { listCommentsOp } from './comment/list-comments'
@@ -49,6 +51,8 @@ export const operations = defineRegistry([
moveAssetsOp,
copyAssetsOp,
deleteAssetOp,
archiveAssetsOp,
restoreArchivedAssetsOp,
readAssetOp,
getAssetRenderOp,
downloadAssetOp,
@@ -0,0 +1,568 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as fs from 'fs'
import * as path from 'path'
import {
PutObjectCommand,
HeadObjectCommand,
CopyObjectCommand,
CreateMultipartUploadCommand,
UploadPartCommand,
UploadPartCopyCommand,
CompleteMultipartUploadCommand,
AbortMultipartUploadCommand,
RestoreObjectCommand,
} from '@aws-sdk/client-s3'
import {
getArchiveStorageConfig,
isArchiveStorageEnabled,
isColdStorageRestoreError,
LocalArchiveStorageService,
parseRestoreHeaderStatus,
resetArchiveStorageServiceForTests,
S3ArchiveStorageService,
} from './archive-storage'
import { uploadStreamToS3Client } from './s3'
const s3ClientConstructorSpy = vi.fn()
const s3SendSpy = vi.fn()
vi.mock('@aws-sdk/client-s3', async (importOriginal) => {
const actual = await importOriginal<typeof import('@aws-sdk/client-s3')>()
return {
...actual,
S3Client: class {
constructor(params: unknown) {
s3ClientConstructorSpy(params)
}
send = s3SendSpy
},
}
})
function makeWebStream(chunks: Array<string | Uint8Array>): ReadableStream<Uint8Array> {
return new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk)
}
controller.close()
},
})
}
async function readAllBytes(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
const reader = stream.getReader()
const parts: Uint8Array[] = []
while (true) {
const { done, value } = await reader.read()
if (done) break
if (value) parts.push(value)
}
return Buffer.concat(parts)
}
describe('Archive Storage', () => {
let originalEnv: NodeJS.ProcessEnv
beforeEach(() => {
originalEnv = { ...process.env }
vi.clearAllMocks()
resetArchiveStorageServiceForTests()
delete process.env.ARCHIVE_STORAGE_BACKEND
delete process.env.ARCHIVE_LOCAL_ROOT
delete process.env.ARCHIVE_S3_BUCKET
delete process.env.ARCHIVE_AWS_ENDPOINT_URL_S3
delete process.env.ARCHIVE_S3_REGION
delete process.env.ARCHIVE_S3_ACCESS_KEY_ID
delete process.env.ARCHIVE_S3_SECRET_ACCESS_KEY
delete process.env.ARCHIVE_S3_FORCE_PATH_STYLE
delete process.env.ARCHIVE_S3_STORAGE_CLASS
delete process.env.ARCHIVE_S3_USE_RESTORE_OBJECT
delete process.env.ARCHIVE_S3_RESTORE_DAYS
delete process.env.ARCHIVE_S3_RESTORE_TIER
})
afterEach(() => {
process.env = originalEnv
resetArchiveStorageServiceForTests()
})
describe('Environment configuration & validation', () => {
it('is disabled when ARCHIVE_STORAGE_BACKEND is unset or empty', () => {
expect(isArchiveStorageEnabled()).toBe(false)
expect(getArchiveStorageConfig()).toEqual({ enabled: false })
process.env.ARCHIVE_STORAGE_BACKEND = ' '
expect(isArchiveStorageEnabled()).toBe(false)
expect(getArchiveStorageConfig()).toEqual({ enabled: false })
})
it('rejects unsupported ARCHIVE_STORAGE_BACKEND values', () => {
process.env.ARCHIVE_STORAGE_BACKEND = 'ftp'
expect(() => getArchiveStorageConfig()).toThrow(/Invalid ARCHIVE_STORAGE_BACKEND/)
})
it('requires ARCHIVE_LOCAL_ROOT for local archive backend and rejects overlap with main local storage', () => {
process.env.STORAGE_BACKEND = 'local'
process.env.S3_BUCKET = 'shumai'
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_LOCAL_ROOT is required/)
process.env.ARCHIVE_LOCAL_ROOT = 'data/shumai'
expect(() => getArchiveStorageConfig()).toThrow(/must not overlap/)
process.env.ARCHIVE_LOCAL_ROOT = 'data/shumai/subdir'
expect(() => getArchiveStorageConfig()).toThrow(/must not overlap/)
process.env.ARCHIVE_LOCAL_ROOT = 'data'
expect(() => getArchiveStorageConfig()).toThrow(/must not overlap/)
process.env.ARCHIVE_LOCAL_ROOT = 'data-archive'
const cfg = getArchiveStorageConfig()
expect(cfg).toEqual({
enabled: true,
backend: 'local',
localRoot: path.resolve('data-archive'),
isSameS3Target: false,
})
})
it('never falls back to main S3_* environment variables when ARCHIVE_STORAGE_BACKEND is s3', () => {
process.env.STORAGE_BACKEND = 's3'
process.env.S3_BUCKET = 'main-bucket'
process.env.S3_ACCESS_KEY_ID = 'main-key'
process.env.S3_SECRET_ACCESS_KEY = 'main-secret'
process.env.AWS_ENDPOINT_URL_S3 = 'http://main-s3:9000'
process.env.ARCHIVE_STORAGE_BACKEND = 's3'
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_BUCKET is required/)
process.env.ARCHIVE_S3_BUCKET = 'archive-bucket'
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_ACCESS_KEY_ID is required/)
process.env.ARCHIVE_S3_ACCESS_KEY_ID = 'archive-key'
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_SECRET_ACCESS_KEY is required/)
process.env.ARCHIVE_S3_SECRET_ACCESS_KEY = 'archive-secret'
const cfg = getArchiveStorageConfig()
expect(cfg).toMatchObject({
enabled: true,
backend: 's3',
bucket: 'archive-bucket',
endpoint: undefined,
accessKeyId: 'archive-key',
secretAccessKey: 'archive-secret',
isSameS3Target: false,
})
})
it('detects same-bucket S3 configuration and requires a non-STANDARD ARCHIVE_S3_STORAGE_CLASS', () => {
process.env.STORAGE_BACKEND = 's3'
process.env.S3_BUCKET = 'shared-bucket'
process.env.AWS_ENDPOINT_URL_S3 = 'https://s3.example.com/'
process.env.ARCHIVE_STORAGE_BACKEND = 's3'
process.env.ARCHIVE_S3_BUCKET = 'shared-bucket'
process.env.ARCHIVE_AWS_ENDPOINT_URL_S3 = 'https://s3.example.com'
process.env.ARCHIVE_S3_ACCESS_KEY_ID = 'ak'
process.env.ARCHIVE_S3_SECRET_ACCESS_KEY = 'sk'
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_STORAGE_CLASS must be set/)
process.env.ARCHIVE_S3_STORAGE_CLASS = 'standard'
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_STORAGE_CLASS must be set/)
process.env.ARCHIVE_S3_STORAGE_CLASS = 'GLACIER_IR'
const cfg = getArchiveStorageConfig()
expect(cfg).toMatchObject({
enabled: true,
backend: 's3',
bucket: 'shared-bucket',
storageClass: 'GLACIER_IR',
isSameS3Target: true,
})
})
})
describe('LocalArchiveStorageService', () => {
const testRoot = path.join(process.cwd(), 'data-test-archive-local')
beforeEach(() => {
if (fs.existsSync(testRoot)) {
fs.rmSync(testRoot, { recursive: true, force: true })
}
})
afterEach(() => {
if (fs.existsSync(testRoot)) {
fs.rmSync(testRoot, { recursive: true, force: true })
}
})
it('stores files directly under <ARCHIVE_LOCAL_ROOT>/<key> without a bucket subfolder and cleans up empty parent folders on delete', async () => {
const localArchive = new LocalArchiveStorageService(testRoot)
const key = 'files/01M5P70NR77MCJRHDZK989S3H2/sample.mp4'
const payload = 'raw video bytes content'
await localArchive.uploadStream(key, makeWebStream([payload]), payload.length, 'video/mp4')
const expectedDiskPath = path.join(
testRoot,
'files',
'01M5P70NR77MCJRHDZK989S3H2',
'sample.mp4',
)
expect(fs.existsSync(expectedDiskPath)).toBe(true)
const head = await localArchive.headObject(key)
expect(head.size).toBe(payload.length)
expect(head.key).toBe(key)
const { stream, contentLength } = await localArchive.getObjectStream(key)
expect(contentLength).toBe(payload.length)
const readBack = await readAllBytes(stream)
expect(readBack.toString('utf8')).toBe(payload)
const restoreStatus = await localArchive.checkOrInitiateObjectRestore(key)
expect(restoreStatus).toEqual({ status: 'ready' })
const deleted = await localArchive.deleteObject(key)
expect(deleted).toBe(1)
expect(fs.existsSync(expectedDiskPath)).toBe(false)
// Empty <root>/files/<ulid> directory should also be cleaned up
expect(fs.existsSync(path.dirname(expectedDiskPath))).toBe(false)
})
it('prevents path traversal outside ARCHIVE_LOCAL_ROOT', async () => {
const localArchive = new LocalArchiveStorageService(testRoot)
await expect(
localArchive.uploadStream('../escape.txt', makeWebStream(['bad']), 3),
).rejects.toThrow(/path traversal/)
})
})
describe('S3ArchiveStorageService', () => {
const baseS3Config = {
enabled: true as const,
backend: 's3' as const,
bucket: 'archive-bucket',
endpoint: 'http://archive-s3:9000',
region: 'us-east-1',
accessKeyId: 'archive-ak',
secretAccessKey: 'archive-sk',
forcePathStyle: true,
storageClass: 'GLACIER_IR',
useRestoreObject: 'auto' as const,
restoreDays: 3,
restoreTier: undefined,
isSameS3Target: false,
}
it('uploads small streams via PutObjectCommand with configured arbitrary StorageClass', async () => {
s3SendSpy.mockResolvedValueOnce({})
const service = new S3ArchiveStorageService({
...baseS3Config,
storageClass: 'ColdArchive',
})
await service.uploadStream(
'files/01ABC/video.mp4',
makeWebStream(['hello archive']),
13,
'video/mp4',
)
expect(s3SendSpy).toHaveBeenCalledTimes(1)
const cmd = s3SendSpy.mock.calls[0][0]
expect(cmd).toBeInstanceOf(PutObjectCommand)
expect(cmd.input).toMatchObject({
Bucket: 'archive-bucket',
Key: 'files/01ABC/video.mp4',
ContentLength: 13,
ContentType: 'video/mp4',
StorageClass: 'ColdArchive',
})
})
it('streams large uploads via multipart upload and aborts on part failure', async () => {
// Test multipart helper with custom thresholds (singlePutMaxBytes = 10, partSizeBytes = 5)
const fakeClient = { send: s3SendSpy } as unknown as import('@aws-sdk/client-s3').S3Client
s3SendSpy
.mockResolvedValueOnce({ UploadId: 'mp-1' }) // CreateMultipartUpload
// eslint-disable-next-line @typescript-eslint/naming-convention
.mockResolvedValueOnce({ ETag: '"etag-1"' }) // Part 1
// eslint-disable-next-line @typescript-eslint/naming-convention
.mockResolvedValueOnce({ ETag: '"etag-2"' }) // Part 2
// eslint-disable-next-line @typescript-eslint/naming-convention
.mockResolvedValueOnce({ ETag: '"etag-3"' }) // Part 3
.mockResolvedValueOnce({}) // CompleteMultipartUpload
await uploadStreamToS3Client(
fakeClient,
{
bucket: 'archive-bucket',
key: 'files/01ABC/large.bin',
stream: makeWebStream(['01234', '56789', 'ab']),
contentLength: 12,
contentType: 'application/octet-stream',
storageClass: 'DEEP_ARCHIVE',
},
5,
10,
)
expect(s3SendSpy).toHaveBeenCalledTimes(5)
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(CreateMultipartUploadCommand)
expect(s3SendSpy.mock.calls[0][0].input).toMatchObject({
Bucket: 'archive-bucket',
Key: 'files/01ABC/large.bin',
StorageClass: 'DEEP_ARCHIVE',
})
expect(s3SendSpy.mock.calls[1][0]).toBeInstanceOf(UploadPartCommand)
expect(s3SendSpy.mock.calls[2][0]).toBeInstanceOf(UploadPartCommand)
expect(s3SendSpy.mock.calls[3][0]).toBeInstanceOf(UploadPartCommand)
expect(s3SendSpy.mock.calls[4][0]).toBeInstanceOf(CompleteMultipartUploadCommand)
expect(s3SendSpy.mock.calls[4][0].input.MultipartUpload.Parts).toEqual([
// eslint-disable-next-line @typescript-eslint/naming-convention
{ ETag: '"etag-1"', PartNumber: 1 },
// eslint-disable-next-line @typescript-eslint/naming-convention
{ ETag: '"etag-2"', PartNumber: 2 },
// eslint-disable-next-line @typescript-eslint/naming-convention
{ ETag: '"etag-3"', PartNumber: 3 },
])
// Verify abort on failure
s3SendSpy.mockReset()
s3SendSpy
.mockResolvedValueOnce({ UploadId: 'mp-fail' })
.mockRejectedValueOnce(new Error('Part upload failed'))
.mockResolvedValueOnce({}) // AbortMultipartUpload
await expect(
uploadStreamToS3Client(
fakeClient,
{
bucket: 'archive-bucket',
key: 'files/01ABC/large.bin',
stream: makeWebStream(['0123456789ab']),
contentLength: 12,
},
5,
10,
),
).rejects.toThrow('Part upload failed')
expect(s3SendSpy.mock.calls[2][0]).toBeInstanceOf(AbortMultipartUploadCommand)
expect(s3SendSpy.mock.calls[2][0].input).toMatchObject({
Bucket: 'archive-bucket',
Key: 'files/01ABC/large.bin',
UploadId: 'mp-fail',
})
})
it('handles checkOrInitiateObjectRestore in auto mode for synchronous vs asynchronous cold storage classes', async () => {
// 1. GLACIER_IR is immediately readable -> ready without calling RestoreObjectCommand
s3SendSpy.mockResolvedValueOnce({
ContentLength: 100,
StorageClass: 'GLACIER_IR',
})
const glacierIrService = new S3ArchiveStorageService({
...baseS3Config,
storageClass: 'GLACIER_IR',
})
expect(await glacierIrService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
status: 'ready',
})
expect(s3SendSpy).toHaveBeenCalledTimes(1)
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(HeadObjectCommand)
// 2. GLACIER with no Restore header -> sends RestoreObjectCommand without GlacierJobParameters by default
s3SendSpy.mockReset()
s3SendSpy
.mockResolvedValueOnce({
ContentLength: 100,
StorageClass: 'GLACIER',
})
.mockResolvedValueOnce({})
const glacierService = new S3ArchiveStorageService({
...baseS3Config,
storageClass: 'GLACIER',
})
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
status: 'restoring',
})
expect(s3SendSpy).toHaveBeenCalledTimes(2)
expect(s3SendSpy.mock.calls[1][0]).toBeInstanceOf(RestoreObjectCommand)
expect(s3SendSpy.mock.calls[1][0].input).toEqual({
Bucket: 'archive-bucket',
Key: 'files/1/a.mp4',
RestoreRequest: {
Days: 3,
},
})
// 3. When ARCHIVE_S3_RESTORE_TIER is explicitly set, includes GlacierJobParameters
s3SendSpy.mockReset()
s3SendSpy
.mockResolvedValueOnce({
ContentLength: 100,
StorageClass: 'DEEP_ARCHIVE',
})
.mockResolvedValueOnce({})
const tieredService = new S3ArchiveStorageService({
...baseS3Config,
storageClass: 'DEEP_ARCHIVE',
restoreTier: 'Standard',
})
expect(await tieredService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
status: 'restoring',
})
expect(s3SendSpy.mock.calls[1][0].input).toEqual({
Bucket: 'archive-bucket',
Key: 'files/1/a.mp4',
RestoreRequest: {
Days: 3,
GlacierJobParameters: { Tier: 'Standard' },
},
})
// 4. Ongoing restore header -> returns 'restoring' without re-sending RestoreObject
s3SendSpy.mockReset()
s3SendSpy.mockResolvedValueOnce({
ContentLength: 100,
StorageClass: 'GLACIER',
Restore: 'ongoing-request="true"',
})
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
status: 'restoring',
})
expect(s3SendSpy).toHaveBeenCalledTimes(1)
// 5. Completed restore header -> returns 'ready'
s3SendSpy.mockReset()
s3SendSpy.mockResolvedValueOnce({
ContentLength: 100,
StorageClass: 'GLACIER',
Restore: 'ongoing-request="false", expiry-date="Wed, 04 Nov 2026 00:00:00 GMT"',
})
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
status: 'ready',
})
// 6. RestoreAlreadyInProgress (409) is treated as idempotent success
s3SendSpy.mockReset()
const conflictErr = Object.assign(new Error('RestoreAlreadyInProgress'), {
name: 'RestoreAlreadyInProgress',
$metadata: { httpStatusCode: 409 },
})
s3SendSpy
.mockResolvedValueOnce({
ContentLength: 100,
StorageClass: 'GLACIER',
})
.mockRejectedValueOnce(conflictErr)
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
status: 'restoring',
})
})
it('rewrites StorageClass in-place via single CopyObject or multipart UploadPartCopy', async () => {
const service = new S3ArchiveStorageService(
{
...baseS3Config,
isSameS3Target: true,
},
{ singleCopyMaxBytes: 100, multipartCopyPartBytes: 60 },
)
// Single CopyObject (<= 100 bytes)
s3SendSpy.mockResolvedValueOnce({})
await service.rewriteObjectStorageClassInPlace(
'files/01ABC/my video #1.mp4',
'GLACIER_IR',
80,
'video/mp4',
)
expect(s3SendSpy).toHaveBeenCalledTimes(1)
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(CopyObjectCommand)
expect(s3SendSpy.mock.calls[0][0].input).toEqual({
Bucket: 'archive-bucket',
Key: 'files/01ABC/my video #1.mp4',
CopySource: 'archive-bucket/files/01ABC/my%20video%20%231.mp4',
StorageClass: 'GLACIER_IR',
MetadataDirective: 'COPY',
ContentType: 'video/mp4',
})
// Multipart UploadPartCopy (> 100 bytes: 130 bytes across 60-byte parts -> 0-59, 60-119, 120-129)
s3SendSpy.mockReset()
s3SendSpy
.mockResolvedValueOnce({ UploadId: 'copy-mp-1' })
// eslint-disable-next-line @typescript-eslint/naming-convention
.mockResolvedValueOnce({ CopyPartResult: { ETag: '"p1"' } })
// eslint-disable-next-line @typescript-eslint/naming-convention
.mockResolvedValueOnce({ CopyPartResult: { ETag: '"p2"' } })
// eslint-disable-next-line @typescript-eslint/naming-convention
.mockResolvedValueOnce({ CopyPartResult: { ETag: '"p3"' } })
.mockResolvedValueOnce({})
await service.rewriteObjectStorageClassInPlace(
'files/01ABC/large.mp4',
'STANDARD',
130,
'video/mp4',
)
expect(s3SendSpy).toHaveBeenCalledTimes(5)
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(CreateMultipartUploadCommand)
expect(s3SendSpy.mock.calls[1][0]).toBeInstanceOf(UploadPartCopyCommand)
expect(s3SendSpy.mock.calls[1][0].input.CopySourceRange).toBe('bytes=0-59')
expect(s3SendSpy.mock.calls[2][0].input.CopySourceRange).toBe('bytes=60-119')
expect(s3SendSpy.mock.calls[3][0].input.CopySourceRange).toBe('bytes=120-129')
expect(s3SendSpy.mock.calls[4][0]).toBeInstanceOf(CompleteMultipartUploadCommand)
})
it('detects cold storage restore errors and parses x-amz-restore headers accurately', () => {
expect(parseRestoreHeaderStatus(undefined)).toBe('none')
expect(parseRestoreHeaderStatus('ongoing-request="true"')).toBe('ongoing')
expect(
parseRestoreHeaderStatus(
'ongoing-request="false", expiry-date="Fri, 21 Dec 2026 00:00:00 GMT"',
),
).toBe('restored')
expect(isColdStorageRestoreError({ name: 'InvalidObjectState' })).toBe(true)
expect(isColdStorageRestoreError({ Code: 'RestoreAlreadyInProgress' })).toBe(true)
expect(
isColdStorageRestoreError(
new Error("The operation is not valid for the object's storage class"),
),
).toBe(true)
expect(isColdStorageRestoreError(new Error('NetworkingError'))).toBe(false)
})
it('headObject maps 404/NotFound to NoSuchKey, but rethrows 500 and other transient errors', async () => {
const service = new S3ArchiveStorageService(baseS3Config)
s3SendSpy.mockRejectedValueOnce({ name: 'NotFound', $metadata: { httpStatusCode: 404 } })
await expect(service.headObject('missing.mp4')).rejects.toThrow('NoSuchKey')
const internalErr = new Error('Internal S3 500')
s3SendSpy.mockRejectedValueOnce(internalErr)
await expect(service.headObject('error.mp4')).rejects.toThrow('Internal S3 500')
})
it('deleteObject returns 0 on 404/NotFound, but rethrows non-404 errors', async () => {
const service = new S3ArchiveStorageService(baseS3Config)
s3SendSpy.mockRejectedValueOnce({ name: 'NotFound', $metadata: { httpStatusCode: 404 } })
const res404 = await service.deleteObject('missing.mp4')
expect(res404).toBe(0)
const permErr = new Error('AccessDenied')
s3SendSpy.mockRejectedValueOnce(permErr)
await expect(service.deleteObject('forbidden.mp4')).rejects.toThrow('AccessDenied')
})
})
})
+735
View File
@@ -0,0 +1,735 @@
import { ObjectInfo } from '@shumai/dtos'
import {
S3Client,
GetObjectCommand,
HeadObjectCommand,
DeleteObjectCommand,
CopyObjectCommand,
CreateMultipartUploadCommand,
UploadPartCopyCommand,
CompleteMultipartUploadCommand,
AbortMultipartUploadCommand,
RestoreObjectCommand,
StorageClass,
Tier,
} from '@aws-sdk/client-s3'
import * as crypto from 'crypto'
import * as fs from 'fs'
import * as path from 'path'
import { Readable } from 'stream'
import {
bodyToWebReadableStream,
isS3NotFoundError,
S3StreamObject,
uploadStreamToS3Client,
writeStreamToLocalFile,
} from './s3'
export type ArchiveStorageBackend = 'local' | 's3'
export type ArchiveRestoreObjectMode = 'auto' | 'true' | 'false'
export interface ArchiveObjectInfo extends ObjectInfo {
storageClass?: string
restoreHeader?: string
}
export interface ArchiveStorageConfigDisabled {
enabled: false
}
export interface ArchiveStorageConfigLocal {
enabled: true
backend: 'local'
localRoot: string
isSameS3Target: false
}
export interface ArchiveStorageConfigS3 {
enabled: true
backend: 's3'
bucket: string
endpoint?: string
region: string
accessKeyId: string
secretAccessKey: string
forcePathStyle: boolean
storageClass?: string
useRestoreObject: ArchiveRestoreObjectMode
restoreDays: number
restoreTier?: string
isSameS3Target: boolean
}
export type ArchiveStorageConfig =
ArchiveStorageConfigDisabled | ArchiveStorageConfigLocal | ArchiveStorageConfigS3
export interface ArchiveStorageService {
readonly backend: ArchiveStorageBackend
readonly isSameS3Target: boolean
readonly storageClass?: string
uploadStream: (
key: string,
stream: ReadableStream<Uint8Array>,
size: number,
contentType?: string,
) => Promise<void>
getObjectStream: (key: string) => Promise<S3StreamObject>
headObject: (key: string) => Promise<ArchiveObjectInfo>
deleteObject: (key: string) => Promise<number>
checkOrInitiateObjectRestore: (key: string) => Promise<{ status: 'ready' | 'restoring' }>
initiateObjectRestoreExplicitly: (key: string) => Promise<{ status: 'ready' | 'restoring' }>
rewriteObjectStorageClassInPlace: (
key: string,
targetStorageClass: string,
expectedSize: number,
contentType?: string,
) => Promise<void>
}
/**
* Storage classes across AWS S3 and S3-compatible providers that require asynchronous
* `RestoreObject` + `HeadObject` (`x-amz-restore`) polling before `GetObject` / `CopyObject`.
* Explicitly excludes `GLACIER_IR` (Glacier Instant Retrieval), which is synchronously readable.
*/
export const KNOWN_ASYNC_RESTORE_STORAGE_CLASSES = new Set([
'GLACIER',
'DEEP_ARCHIVE',
'ARCHIVE',
'COLDARCHIVE',
'COLD_ARCHIVE',
'DEEPCOLDARCHIVE',
'DEEP_COLD_ARCHIVE',
'COLD',
])
const SINGLE_COPY_MAX_BYTES = 5 * 1024 * 1024 * 1024 // 5 GB
const MULTIPART_COPY_PART_BYTES = 512 * 1024 * 1024 // 512 MB
function normalizeEndpointForComparison(endpoint?: string): string {
const trimmed = (endpoint || '').trim().toLowerCase().replace(/\/+$/, '')
return trimmed || 'https://s3.amazonaws.com'
}
function isSubPathOrEqual(candidatePath: string, targetPath: string): boolean {
const rel = path.relative(targetPath, candidatePath)
return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))
}
export function encodeCopySourceKey(bucket: string, key: string): string {
const encodedKey = key
.split('/')
.map((segment) => encodeURIComponent(segment))
.join('/')
return `${bucket}/${encodedKey}`
}
export function parseRestoreHeaderStatus(restoreHeader?: string): 'ongoing' | 'restored' | 'none' {
if (!restoreHeader) return 'none'
if (/ongoing-request\s*=\s*"true"/i.test(restoreHeader)) {
return 'ongoing'
}
if (/ongoing-request\s*=\s*"false"/i.test(restoreHeader)) {
return 'restored'
}
return 'none'
}
export function isColdStorageRestoreError(err: unknown): boolean {
if (!err || typeof err !== 'object') return false
const e = err as Record<string, unknown>
const code =
(typeof e.name === 'string' && e.name) ||
(typeof e.Code === 'string' && e.Code) ||
(typeof e.code === 'string' && e.code) ||
''
if (code === 'InvalidObjectState' || code === 'RestoreAlreadyInProgress') {
return true
}
const msg = typeof e.message === 'string' ? e.message : ''
return (
msg.includes('InvalidObjectState') ||
msg.includes('RestoreAlreadyInProgress') ||
msg.includes("operation is not valid for the object's storage class")
)
}
function isRestoreAlreadyInProgressError(err: unknown): boolean {
if (!err || typeof err !== 'object') return false
const e = err as Record<string, unknown>
const metadata = e.$metadata as { httpStatusCode?: number } | undefined
const code =
(typeof e.name === 'string' && e.name) ||
(typeof e.Code === 'string' && e.Code) ||
(typeof e.code === 'string' && e.code) ||
''
if (code === 'RestoreAlreadyInProgress' || metadata?.httpStatusCode === 409) {
return true
}
return typeof e.message === 'string' && e.message.includes('RestoreAlreadyInProgress')
}
export function getArchiveStorageConfig(): ArchiveStorageConfig {
const rawBackend = (process.env.ARCHIVE_STORAGE_BACKEND || '').trim()
if (!rawBackend) {
return { enabled: false }
}
if (rawBackend !== 'local' && rawBackend !== 's3') {
throw new Error(
`Invalid ARCHIVE_STORAGE_BACKEND "${rawBackend}". Expected "local", "s3", or empty.`,
)
}
if (rawBackend === 'local') {
const rawLocalRoot = (process.env.ARCHIVE_LOCAL_ROOT || '').trim()
if (!rawLocalRoot) {
throw new Error(
'ARCHIVE_LOCAL_ROOT is required when ARCHIVE_STORAGE_BACKEND is set to "local".',
)
}
const resolvedArchiveRoot = path.resolve(rawLocalRoot)
const mainBackend = (process.env.STORAGE_BACKEND || 'local').trim()
if (mainBackend !== 's3') {
const mainBaseDir = path.resolve('data')
const mainBucketDir = path.resolve('data', process.env.S3_BUCKET || 'shumai')
if (
resolvedArchiveRoot === mainBaseDir ||
isSubPathOrEqual(resolvedArchiveRoot, mainBucketDir) ||
isSubPathOrEqual(mainBucketDir, resolvedArchiveRoot)
) {
throw new Error(
`ARCHIVE_LOCAL_ROOT (${resolvedArchiveRoot}) must not overlap with main local storage (${mainBucketDir}).`,
)
}
}
return {
enabled: true,
backend: 'local',
localRoot: resolvedArchiveRoot,
isSameS3Target: false,
}
}
// S3 backend — never fall back to main S3_* environment variables
const bucket = (process.env.ARCHIVE_S3_BUCKET || '').trim()
if (!bucket) {
throw new Error('ARCHIVE_S3_BUCKET is required when ARCHIVE_STORAGE_BACKEND is set to "s3".')
}
const accessKeyId = (process.env.ARCHIVE_S3_ACCESS_KEY_ID || '').trim()
if (!accessKeyId) {
throw new Error(
'ARCHIVE_S3_ACCESS_KEY_ID is required when ARCHIVE_STORAGE_BACKEND is set to "s3".',
)
}
const secretAccessKey = (process.env.ARCHIVE_S3_SECRET_ACCESS_KEY || '').trim()
if (!secretAccessKey) {
throw new Error(
'ARCHIVE_S3_SECRET_ACCESS_KEY is required when ARCHIVE_STORAGE_BACKEND is set to "s3".',
)
}
const endpoint = (process.env.ARCHIVE_AWS_ENDPOINT_URL_S3 || '').trim() || undefined
const region = (process.env.ARCHIVE_S3_REGION || '').trim() || 'auto'
const forcePathStyleRaw = (process.env.ARCHIVE_S3_FORCE_PATH_STYLE || 'true').trim().toLowerCase()
const forcePathStyle = forcePathStyleRaw !== 'false' && forcePathStyleRaw !== '0'
const storageClass = (process.env.ARCHIVE_S3_STORAGE_CLASS || '').trim() || undefined
const useRestoreObjectRaw = (process.env.ARCHIVE_S3_USE_RESTORE_OBJECT || 'auto')
.trim()
.toLowerCase()
if (
useRestoreObjectRaw !== 'auto' &&
useRestoreObjectRaw !== 'true' &&
useRestoreObjectRaw !== 'false'
) {
throw new Error(
`Invalid ARCHIVE_S3_USE_RESTORE_OBJECT "${useRestoreObjectRaw}". Expected "auto", "true", or "false".`,
)
}
const restoreDaysRaw = (process.env.ARCHIVE_S3_RESTORE_DAYS || '1').trim()
const restoreDays = Number.parseInt(restoreDaysRaw, 10)
if (!Number.isFinite(restoreDays) || restoreDays < 1) {
throw new Error(
`Invalid ARCHIVE_S3_RESTORE_DAYS "${restoreDaysRaw}". Expected a positive integer.`,
)
}
const restoreTier = (process.env.ARCHIVE_S3_RESTORE_TIER || '').trim() || undefined
const mainBackend = (process.env.STORAGE_BACKEND || 'local').trim()
const mainBucket = (process.env.S3_BUCKET || 'shumai').trim()
const mainEndpoint = (process.env.AWS_ENDPOINT_URL_S3 || '').trim() || undefined
const isSameS3Target =
mainBackend === 's3' &&
mainBucket === bucket &&
normalizeEndpointForComparison(mainEndpoint) === normalizeEndpointForComparison(endpoint)
if (isSameS3Target) {
if (!storageClass || storageClass.toUpperCase() === 'STANDARD') {
throw new Error(
'When ARCHIVE_S3_BUCKET and endpoint match main S3 storage, ARCHIVE_S3_STORAGE_CLASS must be set to a non-STANDARD storage class.',
)
}
}
return {
enabled: true,
backend: 's3',
bucket,
endpoint,
region,
accessKeyId,
secretAccessKey,
forcePathStyle,
storageClass,
useRestoreObject: useRestoreObjectRaw,
restoreDays,
restoreTier,
isSameS3Target,
}
}
export function isArchiveStorageEnabled(): boolean {
const rawBackend = (process.env.ARCHIVE_STORAGE_BACKEND || '').trim()
return rawBackend === 'local' || rawBackend === 's3'
}
export function validateArchiveStorageConfig(): ArchiveStorageConfig {
return getArchiveStorageConfig()
}
export function getArchiveStoragePath(
key: string,
): { path: string; backend: 's3' | 'local' } | null {
const config = getArchiveStorageConfig()
if (!config.enabled) return null
if (config.backend === 's3') {
return { path: `s3://${config.bucket}/${key}`, backend: 's3' }
}
return { path: path.join(config.localRoot, key), backend: 'local' }
}
export class LocalArchiveStorageService implements ArchiveStorageService {
readonly backend = 'local' as const
readonly isSameS3Target = false
readonly storageClass = undefined
private readonly rootPath: string
constructor(rootPath: string) {
this.rootPath = path.resolve(rootPath)
if (!fs.existsSync(this.rootPath)) {
fs.mkdirSync(this.rootPath, { recursive: true })
}
}
private getFilePath(key: string): string {
const normalizedKey = key.replace(/^\/+/, '')
if (!normalizedKey) {
throw new Error('Invalid archive key: key cannot be empty')
}
const filePath = path.resolve(this.rootPath, normalizedKey)
if (!isSubPathOrEqual(filePath, this.rootPath) || filePath === this.rootPath) {
throw new Error('Invalid archive path: potential path traversal detected')
}
return filePath
}
async uploadStream(
key: string,
stream: ReadableStream<Uint8Array>,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
_size: number,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
_contentType?: string,
): Promise<void> {
const filePath = this.getFilePath(key)
await writeStreamToLocalFile(filePath, stream)
}
async getObjectStream(key: string): Promise<S3StreamObject> {
const filePath = this.getFilePath(key)
try {
const stats = await fs.promises.stat(filePath)
const nodeStream = fs.createReadStream(filePath)
return {
stream: Readable.toWeb(nodeStream) as unknown as ReadableStream<Uint8Array>,
contentLength: stats.size,
contentType: Bun.file(filePath).type || 'application/octet-stream',
}
} catch (e: unknown) {
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
throw new Error('NoSuchKey: The specified key does not exist.', { cause: e })
}
throw e
}
}
async headObject(key: string): Promise<ArchiveObjectInfo> {
const filePath = this.getFilePath(key)
try {
const stats = await fs.promises.stat(filePath)
const hash = crypto
.createHash('md5')
.update(filePath + stats.mtimeMs)
.digest('hex')
return {
key,
size: stats.size,
lastModified: stats.mtime,
contentType: Bun.file(key).type || 'application/octet-stream',
eTag: `"${hash}"`,
}
} catch (e: unknown) {
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
throw new Error('NoSuchKey: The specified key does not exist.', { cause: e })
}
throw e
}
}
async deleteObject(key: string): Promise<number> {
const filePath = this.getFilePath(key)
try {
await fs.promises.unlink(filePath)
// Best-effort cleanup of empty parent folder (e.g. `<root>/files/<ulid>`)
const parentDir = path.dirname(filePath)
if (parentDir !== this.rootPath && isSubPathOrEqual(parentDir, this.rootPath)) {
const remaining = await fs.promises.readdir(parentDir).catch(() => ['non-empty'])
if (remaining.length === 0) {
await fs.promises.rmdir(parentDir).catch(() => {})
}
}
return 1
} catch (e: unknown) {
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
return 0
}
throw e
}
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async checkOrInitiateObjectRestore(_key: string): Promise<{ status: 'ready' | 'restoring' }> {
return { status: 'ready' }
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async initiateObjectRestoreExplicitly(_key: string): Promise<{ status: 'ready' | 'restoring' }> {
return { status: 'ready' }
}
async rewriteObjectStorageClassInPlace(): Promise<void> {
throw new Error('rewriteObjectStorageClassInPlace is only supported for S3 archive storage.')
}
}
export class S3ArchiveStorageService implements ArchiveStorageService {
readonly backend = 's3' as const
readonly isSameS3Target: boolean
readonly storageClass?: string
private readonly client: S3Client
private readonly bucket: string
private readonly useRestoreObject: ArchiveRestoreObjectMode
private readonly restoreDays: number
private readonly restoreTier?: string
private readonly singleCopyMaxBytes: number
private readonly multipartCopyPartBytes: number
constructor(
config: ArchiveStorageConfigS3,
options?: { singleCopyMaxBytes?: number; multipartCopyPartBytes?: number },
) {
this.bucket = config.bucket
this.isSameS3Target = config.isSameS3Target
this.storageClass = config.storageClass
this.useRestoreObject = config.useRestoreObject
this.restoreDays = config.restoreDays
this.restoreTier = config.restoreTier
this.singleCopyMaxBytes = options?.singleCopyMaxBytes ?? SINGLE_COPY_MAX_BYTES
this.multipartCopyPartBytes = options?.multipartCopyPartBytes ?? MULTIPART_COPY_PART_BYTES
this.client = new S3Client({
region: config.region,
...(config.endpoint ? { endpoint: config.endpoint } : {}),
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
forcePathStyle: config.forcePathStyle,
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
})
}
async uploadStream(
key: string,
stream: ReadableStream<Uint8Array>,
size: number,
contentType?: string,
): Promise<void> {
await uploadStreamToS3Client(this.client, {
bucket: this.bucket,
key,
stream,
contentLength: size,
contentType,
storageClass: this.storageClass,
})
}
async getObjectStream(key: string): Promise<S3StreamObject> {
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }))
return {
stream: bodyToWebReadableStream(res.Body),
contentLength: res.ContentLength,
contentType: res.ContentType || Bun.file(key).type || 'application/octet-stream',
}
}
async headObject(key: string): Promise<ArchiveObjectInfo> {
try {
const res = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }))
return {
key,
size: res.ContentLength ?? 0,
lastModified: res.LastModified ?? new Date(),
contentType: res.ContentType || 'application/octet-stream',
eTag: res.ETag || '',
storageClass: res.StorageClass,
restoreHeader: res.Restore,
}
} catch (err: unknown) {
if (isS3NotFoundError(err)) {
throw new Error('NoSuchKey: The specified key does not exist.', { cause: err })
}
throw err
}
}
async deleteObject(key: string): Promise<number> {
try {
await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: key }))
return 1
} catch (err: unknown) {
if (isS3NotFoundError(err)) {
return 0
}
throw err
}
}
private async sendRestoreObjectRequest(key: string): Promise<void> {
try {
await this.client.send(
new RestoreObjectCommand({
Bucket: this.bucket,
Key: key,
/* eslint-disable @typescript-eslint/naming-convention */
RestoreRequest: {
Days: this.restoreDays,
...(this.restoreTier
? { GlacierJobParameters: { Tier: this.restoreTier as Tier } }
: {}),
},
/* eslint-enable @typescript-eslint/naming-convention */
}),
)
} catch (err: unknown) {
if (isRestoreAlreadyInProgressError(err)) {
return
}
throw err
}
}
async checkOrInitiateObjectRestore(key: string): Promise<{ status: 'ready' | 'restoring' }> {
if (this.useRestoreObject === 'false') {
return { status: 'ready' }
}
const head = await this.headObject(key)
const restoreState = parseRestoreHeaderStatus(head.restoreHeader)
if (restoreState === 'restored') {
return { status: 'ready' }
}
if (restoreState === 'ongoing') {
return { status: 'restoring' }
}
const effectiveStorageClass = (head.storageClass || this.storageClass || '').toUpperCase()
const requiresRestore =
this.useRestoreObject === 'true' ||
(this.useRestoreObject === 'auto' &&
KNOWN_ASYNC_RESTORE_STORAGE_CLASSES.has(effectiveStorageClass))
if (!requiresRestore) {
return { status: 'ready' }
}
await this.sendRestoreObjectRequest(key)
return { status: 'restoring' }
}
async initiateObjectRestoreExplicitly(key: string): Promise<{ status: 'ready' | 'restoring' }> {
if (this.useRestoreObject === 'false') {
throw new Error(
'Object is in a cold storage class requiring RestoreObject, but ARCHIVE_S3_USE_RESTORE_OBJECT is set to "false".',
)
}
const head = await this.headObject(key)
const restoreState = parseRestoreHeaderStatus(head.restoreHeader)
if (restoreState === 'restored') {
return { status: 'ready' }
}
if (restoreState === 'ongoing') {
return { status: 'restoring' }
}
await this.sendRestoreObjectRequest(key)
return { status: 'restoring' }
}
async rewriteObjectStorageClassInPlace(
key: string,
targetStorageClass: string,
expectedSize: number,
contentType?: string,
): Promise<void> {
const copySource = encodeCopySourceKey(this.bucket, key)
const s3StorageClass = targetStorageClass as StorageClass
if (expectedSize <= this.singleCopyMaxBytes) {
await this.client.send(
new CopyObjectCommand({
Bucket: this.bucket,
Key: key,
CopySource: copySource,
// eslint-disable-next-line @typescript-eslint/naming-convention
StorageClass: s3StorageClass,
// eslint-disable-next-line @typescript-eslint/naming-convention
MetadataDirective: 'COPY',
...(contentType ? { ContentType: contentType } : {}),
}),
)
return
}
const createRes = await this.client.send(
new CreateMultipartUploadCommand({
Bucket: this.bucket,
Key: key,
// eslint-disable-next-line @typescript-eslint/naming-convention
StorageClass: s3StorageClass,
...(contentType ? { ContentType: contentType } : {}),
}),
)
const uploadId = createRes.UploadId
if (!uploadId) {
throw new Error(`Failed to initiate multipart copy for ${this.bucket}/${key}`)
}
// eslint-disable-next-line @typescript-eslint/naming-convention
const completedParts: Array<{ ETag?: string; PartNumber: number }> = []
let partNumber = 1
let offset = 0
try {
while (offset < expectedSize) {
const end = Math.min(offset + this.multipartCopyPartBytes - 1, expectedSize - 1)
const currentPartNumber = partNumber++
const copyPartRes = await this.client.send(
new UploadPartCopyCommand({
Bucket: this.bucket,
Key: key,
UploadId: uploadId,
PartNumber: currentPartNumber,
CopySource: copySource,
// eslint-disable-next-line @typescript-eslint/naming-convention
CopySourceRange: `bytes=${offset}-${end}`,
}),
)
completedParts.push({
// eslint-disable-next-line @typescript-eslint/naming-convention
ETag: copyPartRes.CopyPartResult?.ETag,
PartNumber: currentPartNumber,
})
offset = end + 1
}
await this.client.send(
new CompleteMultipartUploadCommand({
Bucket: this.bucket,
Key: key,
UploadId: uploadId,
/* eslint-disable @typescript-eslint/naming-convention */
MultipartUpload: {
Parts: completedParts,
},
/* eslint-enable @typescript-eslint/naming-convention */
}),
)
} catch (err) {
await this.client
.send(
new AbortMultipartUploadCommand({
Bucket: this.bucket,
Key: key,
UploadId: uploadId,
}),
)
.catch(() => {})
throw err
}
}
}
let cachedService: ArchiveStorageService | null = null
let cachedConfigSignature: string | null = null
function getEnvSignature(): string {
return JSON.stringify({
b: process.env.ARCHIVE_STORAGE_BACKEND,
lr: process.env.ARCHIVE_LOCAL_ROOT,
sb: process.env.ARCHIVE_S3_BUCKET,
se: process.env.ARCHIVE_AWS_ENDPOINT_URL_S3,
sr: process.env.ARCHIVE_S3_REGION,
ak: process.env.ARCHIVE_S3_ACCESS_KEY_ID,
sk: process.env.ARCHIVE_S3_SECRET_ACCESS_KEY,
fp: process.env.ARCHIVE_S3_FORCE_PATH_STYLE,
sc: process.env.ARCHIVE_S3_STORAGE_CLASS,
ro: process.env.ARCHIVE_S3_USE_RESTORE_OBJECT,
rd: process.env.ARCHIVE_S3_RESTORE_DAYS,
rt: process.env.ARCHIVE_S3_RESTORE_TIER,
mb: process.env.STORAGE_BACKEND,
msb: process.env.S3_BUCKET,
mse: process.env.AWS_ENDPOINT_URL_S3,
})
}
export function getArchiveStorageService(): ArchiveStorageService {
const sig = getEnvSignature()
if (cachedService && cachedConfigSignature === sig) {
return cachedService
}
const config = getArchiveStorageConfig()
if (!config.enabled) {
throw new Error('Archive storage is not enabled (ARCHIVE_STORAGE_BACKEND is not configured).')
}
cachedService =
config.backend === 'local'
? new LocalArchiveStorageService(config.localRoot)
: new S3ArchiveStorageService(config)
cachedConfigSignature = sig
return cachedService
}
export function resetArchiveStorageServiceForTests(): void {
cachedService = null
cachedConfigSignature = null
}
+97 -7
View File
@@ -10,7 +10,13 @@ import {
ListPartsCommand,
AbortMultipartUploadCommand,
} from '@aws-sdk/client-s3'
import { buildContentDisposition, LocalStorageService, S3StorageService } from './s3'
import {
buildContentDisposition,
LocalStorageService,
S3StorageService,
signLocalGetUrl,
verifyLocalGetUrlSignature,
} from './s3'
const s3ClientConstructorSpy = vi.fn()
const s3SendSpy = vi.fn()
@@ -710,19 +716,25 @@ describe('S3Service implementations', () => {
it('should generate a local presign URL', async () => {
const url = await localS3.presign('my-bucket', 'dir/file.txt', 'GET')
expect(url).toBe('http://localhost:3000/files/my-bucket/dir/file.txt')
expect(url).toContain('http://localhost:3000/files/my-bucket/dir/file.txt?')
expect(url).toContain('Signature=')
expect(url).toContain('Expires=')
})
it('should generate a local presign URL with download param', async () => {
const url = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true)
expect(url).toBe('http://localhost:3000/files/my-bucket/dir/file.txt?download=1')
expect(url).toContain('http://localhost:3000/files/my-bucket/dir/file.txt?')
expect(url).toContain('Signature=')
expect(url).toContain('Expires=')
expect(url).toContain('download=1')
})
it('should include an encoded filename in the local download URL', async () => {
const url = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true, 'foo bar.png')
expect(url).toBe(
'http://localhost:3000/files/my-bucket/dir/file.txt?download=1&filename=foo%20bar.png',
)
expect(url).toContain('http://localhost:3000/files/my-bucket/dir/file.txt?')
expect(url).toContain('Signature=')
expect(url).toContain('Expires=')
expect(url).toContain('download=1&filename=foo%20bar.png')
})
it('should not append a filename param when none is provided', async () => {
@@ -730,6 +742,23 @@ describe('S3Service implementations', () => {
expect(url).not.toContain('filename=')
})
it('should cache GET presigned URLs using LRU cache', async () => {
const url1 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET')
const url2 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET')
expect(url1).toBe(url2)
const urlOther = await localS3.presign('my-bucket', 'dir/other.txt', 'GET')
expect(urlOther).not.toBe(url1)
})
it('should not cache download URLs', async () => {
const url1 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true)
const url2 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true)
// download URLs include download=1
expect(url1).toContain('download=1')
expect(url2).toContain('download=1')
})
it('should throw an error for unsupported presign methods', async () => {
await expect(localS3.presign('b', 'k', 'POST')).rejects.toThrow()
})
@@ -742,7 +771,9 @@ describe('S3Service implementations', () => {
it('should resolveInput to presigned URL if file does not exist on disk', async () => {
const input = await localS3.resolveInput('my-bucket', 'nonexistent.mp4')
expect(input).toBe('http://localhost:3000/files/my-bucket/nonexistent.mp4')
expect(input).toContain('http://localhost:3000/files/my-bucket/nonexistent.mp4?')
expect(input).toContain('Signature=')
expect(input).toContain('Expires=')
})
})
@@ -832,4 +863,63 @@ describe('S3Service implementations', () => {
).toBe(true)
})
})
describe('signLocalGetUrl and verifyLocalGetUrlSignature', () => {
it('signs and verifies valid local get url', () => {
const { signature, expires } = signLocalGetUrl('my-bucket', 'files/123/file.mp4')
expect(signature).toBeDefined()
expect(expires).toBeGreaterThan(Math.floor(Date.now() / 1000))
const isValid = verifyLocalGetUrlSignature(
'my-bucket',
'files/123/file.mp4',
signature,
String(expires),
)
expect(isValid).toBe(true)
})
it('rejects expired local get url signature', () => {
// 10 seconds in the past
const { signature, expires } = signLocalGetUrl('my-bucket', 'files/123/file.mp4', -10)
const isValid = verifyLocalGetUrlSignature(
'my-bucket',
'files/123/file.mp4',
signature,
String(expires),
)
expect(isValid).toBe(false)
})
it('rejects tampered bucket or key', () => {
const { signature, expires } = signLocalGetUrl('my-bucket', 'files/123/file.mp4')
expect(
verifyLocalGetUrlSignature(
'other-bucket',
'files/123/file.mp4',
signature,
String(expires),
),
).toBe(false)
expect(
verifyLocalGetUrlSignature('my-bucket', 'files/999/other.mp4', signature, String(expires)),
).toBe(false)
})
it('rejects tampered or malformed signature or expires', () => {
expect(
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', 'badsig', '1234567890'),
).toBe(false)
expect(
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', undefined, '1234567890'),
).toBe(false)
expect(
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', 'badsig', undefined),
).toBe(false)
expect(
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', 'badsig', 'not-a-number'),
).toBe(false)
})
})
})
+358 -6
View File
@@ -14,6 +14,7 @@ import {
ListPartsCommand,
CompleteMultipartUploadCommand,
AbortMultipartUploadCommand,
StorageClass,
} from '@aws-sdk/client-s3'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
import * as crypto from 'crypto'
@@ -45,6 +46,48 @@ export function verifyLocalUrlSignature(bucket: string, key: string, signature:
}
}
export const DEFAULT_LOCAL_GET_EXPIRY_SECONDS = 12 * 3600
export function signLocalGetUrl(
bucket: string,
key: string,
expiresInSeconds: number = DEFAULT_LOCAL_GET_EXPIRY_SECONDS,
): { signature: string; expires: number } {
const expires = Math.floor(Date.now() / 1000) + expiresInSeconds
const secret = process.env.BETTER_AUTH_SECRET || 'shumai-local-storage-secret'
const hmac = crypto.createHmac('sha256', secret)
hmac.update(`GET:${bucket}/${key}:${expires}`)
const signature = hmac.digest('hex')
return { signature, expires }
}
export function verifyLocalGetUrlSignature(
bucket: string,
key: string,
signature: string | undefined | null,
expiresStr: string | undefined | null,
): boolean {
if (!signature || !expiresStr) return false
const expires = parseInt(expiresStr, 10)
if (isNaN(expires)) return false
const now = Math.floor(Date.now() / 1000)
if (now > expires) return false
const secret = process.env.BETTER_AUTH_SECRET || 'shumai-local-storage-secret'
const hmac = crypto.createHmac('sha256', secret)
hmac.update(`GET:${bucket}/${key}:${expires}`)
const expected = hmac.digest('hex')
try {
const sigBuf = Buffer.from(signature, 'hex')
const expBuf = Buffer.from(expected, 'hex')
if (sigBuf.length !== expBuf.length) return false
return crypto.timingSafeEqual(sigBuf, expBuf)
} catch {
return false
}
}
/**
* Build a Content-Disposition header value for downloads.
* - Falls back to plain `attachment` when no filename is provided.
@@ -82,6 +125,241 @@ export interface S3Object {
contentType: string
}
export interface S3StreamObject {
stream: ReadableStream<Uint8Array>
contentLength?: number
contentType?: string
}
export interface UploadStreamParams {
bucket: string
key: string
stream: ReadableStream<Uint8Array>
contentLength?: number
contentType?: string
storageClass?: string
}
const SINGLE_PUT_MAX_BYTES = 64 * 1024 * 1024 // 64 MB
const MULTIPART_PART_SIZE_BYTES = 16 * 1024 * 1024 // 16 MB
export function bodyToWebReadableStream(body: unknown): ReadableStream<Uint8Array> {
if (!body) {
throw new Error('NoSuchKey: The specified key returned an empty body.')
}
if (
typeof body === 'object' &&
'transformToWebStream' in body &&
typeof (body as { transformToWebStream: unknown }).transformToWebStream === 'function'
) {
return (
body as { transformToWebStream: () => ReadableStream<Uint8Array> }
).transformToWebStream()
}
if (typeof body === 'object' && 'getReader' in body) {
return body as ReadableStream<Uint8Array>
}
// Convert Node.js Readable or AsyncIterable to Web ReadableStream
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const nodeReadable = 'pipe' in (body as any) ? (body as Readable) : Readable.from(body as any)
return Readable.toWeb(nodeReadable) as unknown as ReadableStream<Uint8Array>
}
export function isS3NotFoundError(err: unknown): boolean {
if (!err || typeof err !== 'object') return false
const r = err as Record<string, unknown>
const name = typeof r.name === 'string' ? r.name : ''
const code = typeof r.code === 'string' ? r.code : ''
const msg = typeof r.message === 'string' ? r.message : ''
const metadata = r.$metadata as { httpStatusCode?: number } | undefined
return (
name === 'NotFound' ||
name === 'NoSuchKey' ||
code === 'NotFound' ||
code === 'NoSuchKey' ||
code === 'ENOENT' ||
metadata?.httpStatusCode === 404 ||
msg.includes('NoSuchKey') ||
msg.includes('NotFound')
)
}
export async function writeStreamToLocalFile(
targetPath: string,
stream: ReadableStream<Uint8Array>,
): Promise<void> {
const dir = path.dirname(targetPath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
const tempPath = `${targetPath}.tmp-${ulid()}`
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const nodeStream = Readable.fromWeb(stream as any)
await pipeline(nodeStream, fs.createWriteStream(tempPath))
await fs.promises.rename(tempPath, targetPath)
} catch (err) {
await fs.promises.unlink(tempPath).catch(() => {})
throw err
}
}
export async function uploadStreamToS3Client(
client: S3Client,
params: UploadStreamParams,
partSizeBytes: number = MULTIPART_PART_SIZE_BYTES,
singlePutMaxBytes: number = SINGLE_PUT_MAX_BYTES,
): Promise<void> {
const { bucket, key, stream, contentLength, contentType, storageClass } = params
// AWS SDK v3 StorageClass is an open string union at runtime; cast safely for arbitrary S3-compatible classes
const s3StorageClass = storageClass ? (storageClass as StorageClass) : undefined
const MAX_S3_PARTS = 10000
const effectivePartSize =
contentLength !== undefined && contentLength > MAX_S3_PARTS * partSizeBytes
? Math.max(partSizeBytes, Math.ceil(contentLength / 9900))
: partSizeBytes
if (contentLength !== undefined && contentLength <= singlePutMaxBytes) {
if (contentLength === 0) {
await stream.cancel().catch(() => {})
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: Buffer.alloc(0),
ContentLength: 0,
ContentType: contentType,
// eslint-disable-next-line @typescript-eslint/naming-convention
StorageClass: s3StorageClass,
}),
)
return
}
// Convert Web ReadableStream to Node stream without buffering into memory
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const nodeStream = Readable.fromWeb(stream as any)
await client.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Body: nodeStream as any,
ContentLength: contentLength,
ContentType: contentType,
// eslint-disable-next-line @typescript-eslint/naming-convention
StorageClass: s3StorageClass,
}),
)
return
}
// Multipart streaming upload for > 64 MB or unknown contentLength
const createRes = await client.send(
new CreateMultipartUploadCommand({
Bucket: bucket,
Key: key,
ContentType: contentType,
// eslint-disable-next-line @typescript-eslint/naming-convention
StorageClass: s3StorageClass,
}),
)
const uploadId = createRes.UploadId
if (!uploadId) {
throw new Error(`Failed to initiate multipart upload for ${bucket}/${key}`)
}
const reader = stream.getReader()
// eslint-disable-next-line @typescript-eslint/naming-convention
const completedParts: Array<{ ETag?: string; PartNumber: number }> = []
let partNumber = 1
const pendingChunks: Uint8Array[] = []
let pendingLength = 0
const flushPart = async (sizeToFlush: number) => {
const partBuffer = Buffer.allocUnsafe(sizeToFlush)
let offset = 0
while (offset < sizeToFlush && pendingChunks.length > 0) {
const chunk = pendingChunks[0]
const remaining = sizeToFlush - offset
if (chunk.byteLength <= remaining) {
partBuffer.set(chunk, offset)
offset += chunk.byteLength
pendingLength -= chunk.byteLength
pendingChunks.shift()
} else {
partBuffer.set(chunk.subarray(0, remaining), offset)
pendingChunks[0] = chunk.subarray(remaining)
offset += remaining
pendingLength -= remaining
}
}
const currentPartNumber = partNumber++
const partRes = await client.send(
new UploadPartCommand({
Bucket: bucket,
Key: key,
UploadId: uploadId,
PartNumber: currentPartNumber,
Body: partBuffer,
ContentLength: partBuffer.byteLength,
}),
)
completedParts.push({
// eslint-disable-next-line @typescript-eslint/naming-convention
ETag: partRes.ETag,
PartNumber: currentPartNumber,
})
}
try {
while (true) {
const { done, value } = await reader.read()
if (done) {
break
}
if (value && value.byteLength > 0) {
pendingChunks.push(value)
pendingLength += value.byteLength
while (pendingLength >= effectivePartSize) {
await flushPart(effectivePartSize)
}
}
}
if (pendingLength > 0 || completedParts.length === 0) {
await flushPart(pendingLength)
}
await client.send(
new CompleteMultipartUploadCommand({
Bucket: bucket,
Key: key,
UploadId: uploadId,
/* eslint-disable @typescript-eslint/naming-convention */
MultipartUpload: {
Parts: completedParts,
},
/* eslint-enable @typescript-eslint/naming-convention */
}),
)
} catch (err) {
await reader.cancel().catch(() => {})
await client
.send(
new AbortMultipartUploadCommand({
Bucket: bucket,
Key: key,
UploadId: uploadId,
}),
)
.catch(() => {})
throw err
} finally {
reader.releaseLock()
}
}
export interface S3Service {
getObjectSize: (bucket: string, key: string) => Promise<number>
putObject: (
@@ -92,6 +370,8 @@ export interface S3Service {
contentType?: string,
) => Promise<void>
getObject: (bucket: string, key: string) => Promise<S3Object>
getObjectStream: (bucket: string, key: string) => Promise<S3StreamObject>
uploadStream: (params: UploadStreamParams) => Promise<void>
copyObject: (
sourceBucket: string,
sourceKey: string,
@@ -112,7 +392,7 @@ export interface S3Service {
download?: boolean,
filename?: string,
internal?: boolean,
/** URL lifetime; defaults to PRESIGNED_UPLOAD_URL_EXPIRES_IN (12h) for PUT, or PRESIGNED_URL_EXPIRES_IN (5h) for GET. Ignored by local storage. */
/** URL lifetime; defaults to PRESIGNED_UPLOAD_URL_EXPIRES_IN (12h) for PUT, or PRESIGNED_URL_EXPIRES_IN (5h) for GET, or DEFAULT_LOCAL_GET_EXPIRY_SECONDS (12h) for local. */
expiresInSeconds?: number,
) => Promise<string>
presignMultipart: (
@@ -122,6 +402,7 @@ export interface S3Service {
) => Promise<{ url: string }>
abortMultipartUpload: (bucket: string, key: string, uploadId: string) => Promise<void>
resolveInput: (bucket: string, key: string) => Promise<string>
getStoragePath: (bucket: string, key: string) => string
}
export class S3StorageService implements S3Service {
@@ -225,6 +506,19 @@ export class S3StorageService implements S3Service {
}
}
async getObjectStream(bucket: string, key: string): Promise<S3StreamObject> {
const res = await this.client.send(new GetObjectCommand({ Bucket: bucket, Key: key }))
return {
stream: bodyToWebReadableStream(res.Body),
contentLength: res.ContentLength,
contentType: res.ContentType || Bun.file(key).type || 'application/octet-stream',
}
}
async uploadStream(params: UploadStreamParams): Promise<void> {
await uploadStreamToS3Client(this.client, params)
}
async copyObject(
sourceBucket: string,
sourceKey: string,
@@ -267,8 +561,11 @@ export class S3StorageService implements S3Service {
try {
await this.client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key }))
return 1
} catch {
return 0
} catch (err: unknown) {
if (isS3NotFoundError(err)) {
return 0
}
throw err
}
}
@@ -302,7 +599,10 @@ export class S3StorageService implements S3Service {
eTag: res.ETag || '',
}
} catch (err: unknown) {
throw new Error(`NoSuchKey: The specified key does not exist.`, { cause: err })
if (isS3NotFoundError(err)) {
throw new Error(`NoSuchKey: The specified key does not exist.`, { cause: err })
}
throw err
}
}
@@ -503,11 +803,16 @@ export class S3StorageService implements S3Service {
async resolveInput(bucket: string, key: string): Promise<string> {
return this.presign(bucket, key, 'GET', false, undefined, true)
}
getStoragePath(bucket: string, key: string): string {
return `s3://${bucket}/${key}`
}
}
export class LocalStorageService implements S3Service {
private basePath: string
private endpoint: string
private presignCache = new LruTtlCache<string, string>(50000)
constructor(endpoint: string, basePath: string = 'data') {
this.endpoint = endpoint.replace(/\/$/, '')
@@ -583,6 +888,29 @@ export class LocalStorageService implements S3Service {
}
}
async getObjectStream(bucket: string, key: string): Promise<S3StreamObject> {
const filePath = this.getFilePath(bucket, key)
try {
const stats = await fs.promises.stat(filePath)
const nodeStream = fs.createReadStream(filePath)
return {
stream: Readable.toWeb(nodeStream) as unknown as ReadableStream<Uint8Array>,
contentLength: stats.size,
contentType: Bun.file(filePath).type || 'application/octet-stream',
}
} catch (e: unknown) {
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
throw new Error(`NoSuchKey: The specified key does not exist.`, { cause: e })
}
throw e
}
}
async uploadStream(params: UploadStreamParams): Promise<void> {
const filePath = this.getFilePath(params.bucket, params.key)
await writeStreamToLocalFile(filePath, params.stream)
}
async copyObject(
sourceBucket: string,
sourceKey: string,
@@ -725,6 +1053,8 @@ export class LocalStorageService implements S3Service {
method: string,
download?: boolean,
filename?: string,
_internal?: boolean,
expiresInSeconds?: number,
): Promise<string> {
if (method !== 'GET' && method !== 'PUT') {
throw new Error(`Invalid method: ${method}`)
@@ -732,13 +1062,31 @@ export class LocalStorageService implements S3Service {
if (method === 'PUT') {
return `${this.endpoint}${signLocalUrl(bucket, key)}`
}
let url = `${this.endpoint}/files/${bucket}/${key}`
const lifetimeSeconds = expiresInSeconds ?? DEFAULT_LOCAL_GET_EXPIRY_SECONDS
const cacheTtlMs = (lifetimeSeconds * 1000) / 2
const cacheKey = `${bucket}/${key}?download=${download || false}&filename=${filename || ''}&expires=${expiresInSeconds ?? 'default'}`
if (method === 'GET' && !download) {
const cached = this.presignCache.get(cacheKey)
if (cached) {
return cached
}
}
const { signature, expires } = signLocalGetUrl(bucket, key, lifetimeSeconds)
let url = `${this.endpoint}/files/${bucket}/${key}?Signature=${signature}&Expires=${expires}`
if (download) {
url += '?download=1'
url += '&download=1'
if (filename) {
url += `&filename=${encodeURIComponent(filename)}`
}
}
if (method === 'GET' && !download) {
this.presignCache.set(cacheKey, url, cacheTtlMs)
}
return url
}
@@ -769,6 +1117,10 @@ export class LocalStorageService implements S3Service {
}
return this.presign(bucket, key, 'GET')
}
getStoragePath(bucket: string, key: string): string {
return this.getFilePath(bucket, key)
}
}
export function getStorageBackend(): 's3' | 'local' {
@@ -57,7 +57,7 @@ export class EmbeddingService {
const candidateAssets = await prisma.asset.findMany({
where: {
isDeleted: false,
status: 'processed',
status: { in: ['processed', 'archived'] },
type: { in: ['file', 'version_stack'] },
project: { teamId: { in: activeTeamIds } },
},
@@ -106,5 +106,34 @@ describe('SqlQueryBuilder', () => {
)
expect(query.values).toEqual(['custom_field"test', 'hello'])
})
it('isArchived eq true matches archived files and version stacks whose latest version is archived', () => {
const builder = new SqlQueryBuilder()
.from(Prisma.sql`assets a`)
.addSearchConditions('AND', [{ field: 'isArchived', operator: 'eq', value: true }])
const query = builder.build()
expect(query.text).toContain(
`a."type" = 'file'::"AssetType" AND a."status" = 'archived'::"AssetStatus"`,
)
})
it('isArchived eq false matches non-archived files and version stacks', () => {
const builder = new SqlQueryBuilder()
.from(Prisma.sql`assets a`)
.addSearchConditions('AND', [{ field: 'isArchived', operator: 'eq', value: false }])
const query = builder.build()
expect(query.text).toContain(
`a."type" = 'file'::"AssetType" AND a."status" != 'archived'::"AssetStatus"`,
)
})
it('throws an error for unsupported isArchived operators', () => {
const builder = new SqlQueryBuilder().from(Prisma.sql`assets a`)
expect(() =>
builder.addSearchConditions('AND', [{ field: 'isArchived', operator: 'gt', value: true }]),
).toThrow('Unsupported operator for isArchived field: gt')
})
})
})
@@ -285,6 +285,42 @@ export class SqlQueryBuilder {
}
}
if (field === 'isArchived' || field === 'is_archived') {
if (operator !== 'eq' && operator !== 'neq') {
throw new Error(`Unsupported operator for isArchived field: ${operator}`)
}
const rawBool = value === true || value === 'true' || value === 1 || value === '1'
const wantArchived = operator === 'neq' ? !rawBool : rawBool
if (wantArchived) {
return Prisma.sql`(
(a."type" = 'file'::"AssetType" AND a."status" = 'archived'::"AssetStatus")
OR (
a."type" = 'version_stack'::"AssetType"
AND (
SELECT v."status" FROM assets v
WHERE v."parent_id" = a."id" AND v."is_deleted" = false
ORDER BY v."sort_index" ASC LIMIT 1
) = 'archived'::"AssetStatus"
)
)`
}
return Prisma.sql`(
(a."type" = 'file'::"AssetType" AND a."status" != 'archived'::"AssetStatus")
OR (
a."type" = 'version_stack'::"AssetType"
AND COALESCE(
(
SELECT v."status"::text FROM assets v
WHERE v."parent_id" = a."id" AND v."is_deleted" = false
ORDER BY v."sort_index" ASC LIMIT 1
),
''
) != 'archived'
)
OR a."type" NOT IN ('file'::"AssetType", 'version_stack'::"AssetType")
)`
}
if (
field === 'createdAt' ||
field === 'updatedAt' ||
+60 -19
View File
@@ -150,11 +150,11 @@ describe('ShareService', () => {
expect(symlink).toBeDefined()
// Test password check
await expect(shareService.verifyPublicAccess(symlink!.id, 'wrong')).rejects.toThrow(
'Invalid password',
)
await expect(
shareService.verifyPublicAccess(symlink!.id, 'secret-password'),
shareService.verifyPublicAccess(shareLink.id, symlink!.id, 'wrong'),
).rejects.toThrow('Invalid password')
await expect(
shareService.verifyPublicAccess(shareLink.id, symlink!.id, 'secret-password'),
).resolves.toBeDefined()
// Test expiration
@@ -162,9 +162,9 @@ describe('ShareService', () => {
where: { id: shareLink.id },
data: { expireAt: new Date(Date.now() - 1000) },
})
await expect(shareService.verifyPublicAccess(symlink!.id, 'secret-password')).rejects.toThrow(
'expired',
)
await expect(
shareService.verifyPublicAccess(shareLink.id, symlink!.id, 'secret-password'),
).rejects.toThrow('expired')
})
it('traces deep ancestors for public access', async () => {
@@ -192,7 +192,7 @@ describe('ShareService', () => {
},
})
await expect(shareService.verifyPublicAccess(deepFile.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, deepFile.id)).resolves.toBeDefined()
})
it('verifies public access via symlink to a folder', async () => {
@@ -220,10 +220,11 @@ describe('ShareService', () => {
await shareService.addAssetToShare(shareLink.id, { assetIds: [realFolder.id] })
// 3. Verify access to the file inside the real folder (which is reached via the symlinked folder)
await expect(shareService.verifyPublicAccess(realFile.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, realFile.id)).resolves.toBeDefined()
})
it('fails public access for non-shared assets', async () => {
const shareLink = await shareService.createShareLink(projectId, { name: 'Empty Share' })
const privateFile = await prisma.asset.create({
data: {
name: 'private.txt',
@@ -232,7 +233,7 @@ describe('ShareService', () => {
projectId,
},
})
await expect(shareService.verifyPublicAccess(privateFile.id)).rejects.toThrow(
await expect(shareService.verifyPublicAccess(shareLink.id, privateFile.id)).rejects.toThrow(
'Asset is not shared',
)
})
@@ -373,8 +374,8 @@ describe('ShareService', () => {
where: { targetId: file.id, parentId: shareLink.rootFolderId },
})
expect(symlink).toBeDefined()
await expect(shareService.verifyPublicAccess(file.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(symlink!.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, file.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, symlink!.id)).resolves.toBeDefined()
// 1. Soft-delete the file (move to trash)
await assetService.deleteAssets([file.id])
@@ -393,8 +394,8 @@ describe('ShareService', () => {
expect(afterDeleteShareRoot?.fileCount).toBe(0)
// Verify: public access is rejected
await expect(shareService.verifyPublicAccess(file.id)).rejects.toThrow()
await expect(shareService.verifyPublicAccess(symlink!.id)).rejects.toThrow()
await expect(shareService.verifyPublicAccess(shareLink.id, file.id)).rejects.toThrow()
await expect(shareService.verifyPublicAccess(shareLink.id, symlink!.id)).rejects.toThrow()
// Verify: project's recently-deleted view does NOT list symlinks
const recentlyDeleted = await assetService.listChildren({
@@ -423,8 +424,8 @@ describe('ShareService', () => {
expect(afterRestoreShareRoot?.fileCount).toBe(1)
// Verify: public access works again
await expect(shareService.verifyPublicAccess(file.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(symlink!.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, file.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, symlink!.id)).resolves.toBeDefined()
})
it('hides soft-deleted folders from share link and restores them upon folder restoration', async () => {
@@ -463,7 +464,7 @@ describe('ShareService', () => {
})
expect(initialChildren.data).toHaveLength(1)
expect(initialChildren.data[0].name).toBe('Shared Folder')
await expect(shareService.verifyPublicAccess(childFile.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, childFile.id)).resolves.toBeDefined()
// 1. Soft-delete the folder
await assetService.deleteAssets([folder.id])
@@ -476,7 +477,7 @@ describe('ShareService', () => {
expect(afterDeleteChildren.data).toHaveLength(0)
// Verify: access to child file inside folder is rejected
await expect(shareService.verifyPublicAccess(childFile.id)).rejects.toThrow()
await expect(shareService.verifyPublicAccess(shareLink.id, childFile.id)).rejects.toThrow()
// 2. Restore the folder
await assetService.restoreAssets([folder.id])
@@ -490,6 +491,46 @@ describe('ShareService', () => {
expect(afterRestoreChildren.data[0].name).toBe('Shared Folder')
// Verify: access to child file inside folder is restored
await expect(shareService.verifyPublicAccess(childFile.id)).resolves.toBeDefined()
await expect(shareService.verifyPublicAccess(shareLink.id, childFile.id)).resolves.toBeDefined()
})
it('binds public access strictly to the requested shareId', async () => {
const fileA = await prisma.asset.create({
data: { name: 'fileA.txt', type: AssetType.file, status: 'processed', projectId },
})
const fileB = await prisma.asset.create({
data: { name: 'fileB.txt', type: AssetType.file, status: 'processed', projectId },
})
const shareOpen = await shareService.createShareLink(projectId, { name: 'Open Share' })
const shareProtected = await shareService.createShareLink(projectId, {
name: 'Protected Share',
password: 'password123',
})
await shareService.addAssetToShare(shareOpen.id, { assetIds: [fileA.id, fileB.id] })
await shareService.addAssetToShare(shareProtected.id, { assetIds: [fileA.id] })
// fileA in protected share requires password
await expect(
shareService.verifyPublicAccess(shareProtected.id, fileA.id, 'wrong'),
).rejects.toThrow('Invalid password')
await expect(
shareService.verifyPublicAccess(shareProtected.id, fileA.id, 'password123'),
).resolves.toBeDefined()
// fileB is only in shareOpen, accessing it via shareProtected must fail even with password
await expect(
shareService.verifyPublicAccess(shareProtected.id, fileB.id, 'password123'),
).rejects.toThrow('Asset is not shared')
// Disabled share must reject access even if file sits in another active share
await prisma.shareLink.update({
where: { id: shareProtected.id },
data: { isDisabled: true },
})
await expect(
shareService.verifyPublicAccess(shareProtected.id, fileA.id, 'password123'),
).rejects.toThrow('disabled')
})
})
+40 -44
View File
@@ -249,57 +249,16 @@ export class ShareService {
}
async verifyPublicAccess(
shareLinkId: string,
assetId: string,
providedPassword?: string,
): Promise<Prisma.ShareLinkGetPayload<true>> {
// 1. Trace ancestors of the current asset in the real tree
// 2. Find if any ancestor (including itself) is the target of a symlink that belongs to a share
// 3. Or if the asset itself is a descendant of a 'share' type folder (direct share link root or subfolder within)
const rows = await prisma.$queryRaw<{ shareRootId: string }[]>`
WITH RECURSIVE
-- Trace ancestors in the real asset tree
real_ancestors AS (
SELECT id, parent_id, type::text FROM assets WHERE id = ${assetId} AND is_deleted = false
UNION ALL
SELECT a.id, a.parent_id, a.type::text FROM assets a
INNER JOIN real_ancestors ra ON ra.parent_id = a.id
WHERE a.is_deleted = false
),
-- Find all symlinks pointing to any of these real ancestors
found_symlinks AS (
SELECT s.id, s.parent_id, s.type::text FROM assets s
INNER JOIN real_ancestors ra ON s.target_id = ra.id
WHERE s.type = 'symlink' AND s.is_deleted = false
),
-- Combine: start with either real ancestors that are 'share' type, or symlinks that lead to 'share' type
search_starts AS (
SELECT id, parent_id, type FROM real_ancestors WHERE type = 'share'
UNION
SELECT id, parent_id, type FROM found_symlinks
),
-- Trace UP from these search starts to find the ultimate 'share' root
share_trace AS (
SELECT id, parent_id, type FROM search_starts
UNION ALL
SELECT a.id, a.parent_id, a.type::text FROM assets a
INNER JOIN share_trace st ON st.parent_id = a.id
WHERE a.is_deleted = false
)
SELECT id as "shareRootId" FROM share_trace WHERE type = 'share' LIMIT 1;
`
if (rows.length === 0) {
throw new ShareLinkNotFoundError('Asset is not shared')
}
const shareRootId = rows[0].shareRootId
const shareLink = await prisma.shareLink.findUnique({
where: { rootFolderId: shareRootId },
where: { id: shareLinkId },
})
if (!shareLink) {
throw new ShareLinkNotFoundError('Share link not found for this asset')
throw new ShareLinkNotFoundError('Share link not found')
}
if (shareLink.isDisabled) {
@@ -314,6 +273,43 @@ export class ShareService {
throw new ShareLinkPasswordInvalidError('Invalid password for share link')
}
const rows = await prisma.$queryRaw<{ id: string }[]>`
WITH RECURSIVE
-- Trace ancestors in the real asset tree
real_ancestors AS (
SELECT id, parent_id, type::text, target_id FROM assets WHERE id = ${assetId} AND is_deleted = false
UNION ALL
SELECT a.id, a.parent_id, a.type::text, a.target_id FROM assets a
INNER JOIN real_ancestors ra ON ra.parent_id = a.id
WHERE a.is_deleted = false
),
-- Find all symlinks pointing to any of these real ancestors
found_symlinks AS (
SELECT s.id, s.parent_id, s.type::text FROM assets s
INNER JOIN real_ancestors ra ON s.target_id = ra.id
WHERE s.type = 'symlink' AND s.is_deleted = false
),
-- Combine: start with either real ancestors, or symlinks that lead to them
search_starts AS (
SELECT id, parent_id, type FROM real_ancestors
UNION
SELECT id, parent_id, type FROM found_symlinks
),
-- Trace UP from these search starts to find the ultimate 'share' root
share_trace AS (
SELECT id, parent_id, type FROM search_starts
UNION ALL
SELECT a.id, a.parent_id, a.type::text FROM assets a
INNER JOIN share_trace st ON st.parent_id = a.id
WHERE a.is_deleted = false
)
SELECT id FROM share_trace WHERE id = ${shareLink.rootFolderId} LIMIT 1;
`
if (rows.length === 0) {
throw new ShareLinkNotFoundError('Asset is not shared')
}
return shareLink
}
+84
View File
@@ -1,3 +1,4 @@
import * as path from 'path'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { teamService } from '@shumai/core/src/team/team'
@@ -388,6 +389,13 @@ describe('TeamService', () => {
username: undefined,
},
appearance: { hideAgent: false },
archiveStorage: { enabled: false },
autoArchive: {
enabled: false,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
})
await teamService.updateSettings(team.id, 'theme', 'dark')
@@ -417,6 +425,13 @@ describe('TeamService', () => {
username: undefined,
},
appearance: { hideAgent: false },
archiveStorage: { enabled: false },
autoArchive: {
enabled: false,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
})
await teamService.updateSettings(team.id, 'transcode.videoStrategy', 'multi')
@@ -456,6 +471,13 @@ describe('TeamService', () => {
},
appearance: { hideAgent: true },
semanticSearchEnabled: false,
archiveStorage: { enabled: false },
autoArchive: {
enabled: false,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
},
})
})
@@ -633,4 +655,66 @@ describe('TeamService', () => {
delete process.env.ENABLE_EMBEDDING_AGENT
}
})
it('rejects enabling auto-archive when archive storage is not configured', async () => {
delete process.env.ARCHIVE_STORAGE_BACKEND
const owner = await prisma.user.create({
data: { name: 'Owner AA', email: 'owner-aa@example.com', password: 'pw' },
})
const team = await teamService.createTeam(owner, { name: 'AA Team' })
await expect(
teamService.updateSettings(team.id, 'autoArchive', {
enabled: true,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
}),
).rejects.toThrow('Cannot enable auto archive when archive storage is not configured')
await expect(teamService.updateSettings(team.id, 'autoArchive.enabled', true)).rejects.toThrow(
'Cannot enable auto archive when archive storage is not configured',
)
})
it('allows updating auto-archive settings when archive storage is configured', async () => {
const saved = { ...process.env }
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
process.env.ARCHIVE_LOCAL_ROOT = '/tmp/shumai-archive-test-team'
try {
const owner = await prisma.user.create({
data: { name: 'Owner AA 2', email: 'owner-aa2@example.com', password: 'pw' },
})
const team = await teamService.createTeam(owner, { name: 'AA Team 2' })
await teamService.updateSettings(team.id, 'autoArchive', {
enabled: true,
method: 'after_created',
days: 45,
postRestoreGraceDays: 15,
})
const settings = await teamService.getSettings(team.id)
expect(settings.autoArchive).toEqual({
enabled: true,
method: 'after_created',
days: 45,
postRestoreGraceDays: 15,
})
expect(settings.archiveStorage).toEqual({
enabled: true,
backend: 'local',
localRoot: path.resolve('/tmp/shumai-archive-test-team'),
})
// Non-admin call should mask backend/root details
const readerSettings = await teamService.getSettings(team.id, { isAdmin: false })
expect(readerSettings.archiveStorage).toEqual({
enabled: true,
})
} finally {
process.env = saved
}
})
})
+81 -1
View File
@@ -10,6 +10,10 @@ import { AgentService, agentService } from '@shumai/core/src/agent/agent'
import { sandboxService } from '@shumai/core/src/sandbox/sandbox-service'
import { turntableService } from '@shumai/core/src/turntable/turntable'
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
import {
getArchiveStorageConfig,
isArchiveStorageEnabled,
} from '@shumai/core/src/s3/archive-storage'
import {
ServiceCreateTeamRequest,
ServiceGetUserTeamsRequest,
@@ -23,6 +27,9 @@ import {
SandboxSettings,
UpdateSandboxSettingsRequest,
TeamSettingsResponse,
AutoArchiveMethod,
AutoArchiveSettings,
DEFAULT_AUTO_ARCHIVE_SETTINGS,
} from '@shumai/dtos'
export class TeamService {
@@ -300,7 +307,10 @@ export class TeamService {
})
}
async getSettings(teamId: string): Promise<TeamSettingsResponse> {
async getSettings(
teamId: string,
options?: { isAdmin?: boolean },
): Promise<TeamSettingsResponse> {
const team = await prisma.team.findUnique({
where: { id: teamId },
})
@@ -355,6 +365,33 @@ export class TeamService {
settings.turntable = await turntableService.getSettings(teamId)
settings.gotenberg = await gotenbergService.getSettings(teamId)
const archiveConfig = getArchiveStorageConfig()
if (!archiveConfig.enabled) {
settings.archiveStorage = { enabled: false }
} else if (options?.isAdmin === false) {
settings.archiveStorage = { enabled: true }
} else if (archiveConfig.backend === 'local') {
settings.archiveStorage = {
enabled: true,
backend: 'local',
localRoot: archiveConfig.localRoot,
}
} else {
settings.archiveStorage = {
enabled: true,
backend: 's3',
endpoint: archiveConfig.endpoint,
bucket: archiveConfig.bucket,
}
}
settings.autoArchive = {
enabled: settings.autoArchive?.enabled ?? false,
method: settings.autoArchive?.method ?? 'not_updated',
days: settings.autoArchive?.days ?? 30,
postRestoreGraceDays: settings.autoArchive?.postRestoreGraceDays ?? 30,
}
return settings
}
@@ -421,6 +458,49 @@ export class TeamService {
}
settings.appearance.hideAgent = Boolean(value)
delete settings['appearance.hideAgent']
} else if (key === 'autoArchive') {
const autoArchiveVal = value as AutoArchiveSettings
if (autoArchiveVal.enabled && !isArchiveStorageEnabled()) {
throw new HTTPException(400, {
message: 'Cannot enable auto archive when archive storage is not configured',
})
}
settings.autoArchive = {
enabled: Boolean(autoArchiveVal.enabled),
method: autoArchiveVal.method || 'not_updated',
days: Number(autoArchiveVal.days) || 30,
postRestoreGraceDays: Number(autoArchiveVal.postRestoreGraceDays) || 30,
}
} else if (key === 'autoArchive.enabled') {
const enabled = Boolean(value)
if (enabled && !isArchiveStorageEnabled()) {
throw new HTTPException(400, {
message: 'Cannot enable auto archive when archive storage is not configured',
})
}
if (!settings.autoArchive) {
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
}
settings.autoArchive.enabled = enabled
delete settings['autoArchive.enabled']
} else if (key === 'autoArchive.method') {
if (!settings.autoArchive) {
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
}
settings.autoArchive.method = value as AutoArchiveMethod
delete settings['autoArchive.method']
} else if (key === 'autoArchive.days') {
if (!settings.autoArchive) {
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
}
settings.autoArchive.days = Number(value)
delete settings['autoArchive.days']
} else if (key === 'autoArchive.postRestoreGraceDays') {
if (!settings.autoArchive) {
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
}
settings.autoArchive.postRestoreGraceDays = Number(value)
delete settings['autoArchive.postRestoreGraceDays']
} else {
settings[key] = value
}
+77
View File
@@ -170,6 +170,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -228,6 +229,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -268,6 +270,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -303,6 +306,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -336,6 +340,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -366,6 +371,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -399,6 +405,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -429,6 +436,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -484,6 +492,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -512,6 +521,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: htmlTask.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -547,6 +557,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -584,6 +595,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -616,6 +628,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -649,6 +662,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -752,6 +766,7 @@ describe('UploadService', () => {
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: stack.id } },
task: { connect: { id: task.id } },
status: AssetStatus.uploading,
storageKey: {
connectOrCreate: {
@@ -1120,6 +1135,68 @@ describe('UploadService', () => {
})
expect(deletedAsset).toBeNull()
})
it('should not increment task count if error arrives for an already finished or deleted asset', async () => {
const task = await prisma.task.create({
data: {
creatorId: userId,
type: 'upload',
name: 'task-late-err',
total: 2,
uploaded: 1,
status: TaskStatus.uploading,
},
})
const storageKey = await prisma.storageKey.create({
data: { key: 'files/test/already-uploaded.mp4' },
})
const asset = await prisma.asset.create({
data: {
name: 'already-uploaded.mp4',
type: AssetType.file,
projectId,
storageKeyId: storageKey.id,
taskId: task.id,
status: AssetStatus.uploaded,
},
})
await uploadService.confirmFileUpload(userId, task.id, {
fileId: asset.id,
errorMessage: 'Late network error',
})
const taskAfter = await prisma.task.findUniqueOrThrow({ where: { id: task.id } })
expect(taskAfter.uploaded).toBe(1)
})
it('should reject confirmation if asset does not belong to specified task', async () => {
const task1 = await prisma.task.create({
data: { creatorId: userId, type: 'upload', total: 1, status: TaskStatus.uploading },
})
const task2 = await prisma.task.create({
data: { creatorId: userId, type: 'upload', total: 1, status: TaskStatus.uploading },
})
const storageKey = await prisma.storageKey.create({
data: { key: 'files/test/task-mismatch.mp4' },
})
const asset = await prisma.asset.create({
data: {
name: 'task-mismatch.mp4',
type: AssetType.file,
projectId,
storageKeyId: storageKey.id,
taskId: task1.id,
status: AssetStatus.uploading,
},
})
await expect(
uploadService.confirmFileUpload(userId, task2.id, { fileId: asset.id }),
).rejects.toThrow('Asset does not belong to specified task')
})
})
describe('cancelUploadTask', () => {
+30 -16
View File
@@ -206,23 +206,28 @@ export class UploadService {
},
})
if (!asset) throw new Error('Asset not found')
if (asset.taskId !== taskId) {
throw new Error('Asset does not belong to specified task')
}
if (!asset.project) throw new Error('Project not found for asset')
const key = asset.storageKey?.key
if (!key) throw new Error('Asset has no key')
if (req.errorMessage) {
await this.discardUploadingAsset(asset)
const t = await this.prismaClient.task.findUnique({ where: { id: taskId } })
if (t) {
const updatedTask = await this.prismaClient.task.update({
where: { id: taskId },
data: { uploaded: { increment: 1 } },
})
if (updatedTask.uploaded === updatedTask.total) {
await this.prismaClient.task.update({
const discarded = await this.discardUploadingAsset(asset)
if (discarded) {
const t = await this.prismaClient.task.findUnique({ where: { id: taskId } })
if (t) {
const updatedTask = await this.prismaClient.task.update({
where: { id: taskId },
data: { status: TaskStatus.completed },
data: { uploaded: { increment: 1 } },
})
if (updatedTask.uploaded === updatedTask.total) {
await this.prismaClient.task.update({
where: { id: taskId },
data: { status: TaskStatus.completed },
})
}
}
}
return
@@ -242,15 +247,22 @@ export class UploadService {
resolvedMediaType = Bun.file(asset.name).type || 'application/octet-stream'
}
// Update asset
const updatedAsset = await tx.asset.update({
where: { id: asset.id },
// Claim status transition atomically
const claim = await tx.asset.updateMany({
where: {
id: asset.id,
taskId: taskId,
status: AssetStatus.uploading,
},
data: {
status: AssetStatus.uploaded,
sizeByte: size,
mediaType: resolvedMediaType,
},
})
if (claim.count !== 1) {
throw new Error('Asset is not in uploading status')
}
// Update parent and ancestors
if (asset.parentId) {
@@ -259,7 +271,7 @@ export class UploadService {
data: { fileCount: { increment: 1 } },
})
await assetService.updateAncestorsSize(tx, asset.parentId, updatedAsset.sizeByte)
await assetService.updateAncestorsSize(tx, asset.parentId, size)
}
// Update task
@@ -533,7 +545,7 @@ export class UploadService {
storageKey: { key: string } | null
},
requestedUploadId?: string,
): Promise<void> {
): Promise<boolean> {
const deleteResult = await this.prismaClient.asset.deleteMany({
where: {
id: asset.id,
@@ -542,7 +554,7 @@ export class UploadService {
})
if (deleteResult.count === 0) {
return
return false
}
const bucket = process.env.S3_BUCKET || 'shumai'
@@ -568,6 +580,8 @@ export class UploadService {
.delete({ where: { id: asset.storageKeyId } })
.catch(() => {})
}
return true
}
async cancelUploadTask(
@@ -547,4 +547,41 @@ describe('WorkflowTaskService', () => {
expect(stopRes.task.fileSize).toBe(5242880)
expect(stopRes.task.projectName).toBe('Enrich Project')
})
it('reverts archiving and restoring asset status back to baseline on stopWorkflowTask', async () => {
const team = await prisma.team.create({ data: { name: 'Stop Task Team' } })
const project = await prisma.project.create({
data: { name: 'Stop Project', teamId: team.id },
})
const sk = await prisma.storageKey.create({
data: { key: 'keys/stop-test.mp4', isArchived: false },
})
const asset = await prisma.asset.create({
data: {
name: 'stop-test.mp4',
type: 'file',
status: AssetStatus.archiving,
storageKeyId: sk.id,
projectId: project.id,
},
})
const task = await prisma.workflowTask.create({
data: {
assetId: asset.id,
teamId: team.id,
projectId: project.id,
type: WorkflowTaskType.archive_asset,
status: WorkflowTaskStatus.processing,
payload: { projectId: project.id, archive: { storageKeyId: sk.id } },
},
})
await workflowTaskService.stopWorkflowTask({ teamId: team.id, taskId: task.id })
const updatedAsset = await prisma.asset.findUniqueOrThrow({ where: { id: asset.id } })
expect(updatedAsset.status).toBe(AssetStatus.processed)
const updatedTask = await prisma.workflowTask.findUniqueOrThrow({ where: { id: task.id } })
expect(updatedTask.status).toBe(WorkflowTaskStatus.failed)
})
})
+25 -9
View File
@@ -8,6 +8,7 @@ import {
import type { WorkflowTaskItem, ListWorkflowTasksResponse } from '@shumai/dtos'
import { paginateQuery } from '@shumai/core/src/pagination'
import { logger } from '@shumai/core/src/logger'
import { revertArchiveStateForStorageKey } from '@shumai/core/src/asset/asset-archive'
import { HTTPException } from 'hono/http-exception'
export interface ListWorkflowTasksParams {
@@ -300,15 +301,30 @@ export class WorkflowTaskService {
}
if (task.assetId) {
await tx.$executeRaw`
UPDATE "assets"
SET
"status" = 'failed'::"AssetStatus",
"media" = COALESCE("media", '{}'::jsonb) || jsonb_build_object('error', ${errorMsg}::text)
WHERE "id" = ${task.assetId}
AND "is_deleted" = false
AND "status" != 'processed'::"AssetStatus"
`
if (task.type === 'archive_asset' || task.type === 'restore_asset') {
const payload = task.payload as PrismaJson.WorkflowTaskPayload | null
let storageKeyId = payload?.archive?.storageKeyId
if (!storageKeyId) {
const targetAsset = await tx.asset.findUnique({
where: { id: task.assetId },
select: { storageKeyId: true },
})
storageKeyId = targetAsset?.storageKeyId ?? undefined
}
if (storageKeyId) {
await revertArchiveStateForStorageKey(storageKeyId, tx)
}
} else {
await tx.$executeRaw`
UPDATE "assets"
SET
"status" = 'failed'::"AssetStatus",
"media" = COALESCE("media", '{}'::jsonb) || jsonb_build_object('error', ${errorMsg}::text)
WHERE "id" = ${task.assetId}
AND "is_deleted" = false
AND "status" NOT IN ('processed'::"AssetStatus", 'archived'::"AssetStatus", 'archiving'::"AssetStatus", 'restoring'::"AssetStatus")
`
}
}
return await tx.workflowTask.findUniqueOrThrow({
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/db",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -0,0 +1,25 @@
-- AlterEnum
-- This migration adds more than one value to an enum.
-- With PostgreSQL versions 11 and earlier, this is not possible
-- in a single migration. This can be worked around by creating
-- multiple migrations, each migration adding only one value to
-- the enum.
ALTER TYPE "AssetStatus" ADD VALUE 'archiving';
ALTER TYPE "AssetStatus" ADD VALUE 'archived';
ALTER TYPE "AssetStatus" ADD VALUE 'restoring';
-- AlterEnum
-- This migration adds more than one value to an enum.
-- With PostgreSQL versions 11 and earlier, this is not possible
-- in a single migration. This can be worked around by creating
-- multiple migrations, each migration adding only one value to
-- the enum.
ALTER TYPE "WorkflowTaskType" ADD VALUE 'archive_asset';
ALTER TYPE "WorkflowTaskType" ADD VALUE 'restore_asset';
-- AlterTable
ALTER TABLE "storage_keys" ADD COLUMN "is_archived" BOOLEAN NOT NULL DEFAULT false;
+12 -6
View File
@@ -40,6 +40,9 @@ enum AssetStatus {
uploaded
processing
processed
archiving
archived
restoring
failed
trashed
pending_purge
@@ -128,6 +131,8 @@ enum WorkflowTaskType {
query_embedding_for_search
agent_tool_call // @deprecated: Agent tools now execute directly in-process via Core services
kanban_agent_run
archive_asset
restore_asset
}
enum WorkflowTaskStatus {
@@ -606,12 +611,13 @@ model Asset {
}
model StorageKey {
id String @id @default(ulid())
key String @unique
status StorageKeyStatus @default(active)
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
assets Asset[]
id String @id @default(ulid())
key String @unique
status StorageKeyStatus @default(active)
isArchived Boolean @default(false) @map("is_archived")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
assets Asset[]
@@map("storage_keys")
}
+6 -1
View File
@@ -36,6 +36,9 @@ export const AssetStatus = {
uploaded: 'uploaded',
processing: 'processing',
processed: 'processed',
archiving: 'archiving',
archived: 'archived',
restoring: 'restoring',
failed: 'failed',
trashed: 'trashed',
pending_purge: 'pending_purge'
@@ -156,7 +159,9 @@ export const WorkflowTaskType = {
ai_embedding: 'ai_embedding',
query_embedding_for_search: 'query_embedding_for_search',
agent_tool_call: 'agent_tool_call',
kanban_agent_run: 'kanban_agent_run'
kanban_agent_run: 'kanban_agent_run',
archive_asset: 'archive_asset',
restore_asset: 'restore_asset'
} as const
export type WorkflowTaskType = (typeof WorkflowTaskType)[keyof typeof WorkflowTaskType]
File diff suppressed because one or more lines are too long
@@ -5201,6 +5201,7 @@ export const StorageKeyScalarFieldEnum = {
id: 'id',
key: 'key',
status: 'status',
isArchived: 'isArchived',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
@@ -375,6 +375,7 @@ export const StorageKeyScalarFieldEnum = {
id: 'id',
key: 'key',
status: 'status',
isArchived: 'isArchived',
createdAt: 'createdAt',
updatedAt: 'updatedAt'
} as const
+33 -1
View File
@@ -30,6 +30,7 @@ export type StorageKeyMinAggregateOutputType = {
id: string | null
key: string | null
status: $Enums.StorageKeyStatus | null
isArchived: boolean | null
createdAt: Date | null
updatedAt: Date | null
}
@@ -38,6 +39,7 @@ export type StorageKeyMaxAggregateOutputType = {
id: string | null
key: string | null
status: $Enums.StorageKeyStatus | null
isArchived: boolean | null
createdAt: Date | null
updatedAt: Date | null
}
@@ -46,6 +48,7 @@ export type StorageKeyCountAggregateOutputType = {
id: number
key: number
status: number
isArchived: number
createdAt: number
updatedAt: number
_all: number
@@ -56,6 +59,7 @@ export type StorageKeyMinAggregateInputType = {
id?: true
key?: true
status?: true
isArchived?: true
createdAt?: true
updatedAt?: true
}
@@ -64,6 +68,7 @@ export type StorageKeyMaxAggregateInputType = {
id?: true
key?: true
status?: true
isArchived?: true
createdAt?: true
updatedAt?: true
}
@@ -72,6 +77,7 @@ export type StorageKeyCountAggregateInputType = {
id?: true
key?: true
status?: true
isArchived?: true
createdAt?: true
updatedAt?: true
_all?: true
@@ -153,6 +159,7 @@ export type StorageKeyGroupByOutputType = {
id: string
key: string
status: $Enums.StorageKeyStatus
isArchived: boolean
createdAt: Date
updatedAt: Date
_count: StorageKeyCountAggregateOutputType | null
@@ -182,6 +189,7 @@ export type StorageKeyWhereInput = {
id?: Prisma.StringFilter<"StorageKey"> | string
key?: Prisma.StringFilter<"StorageKey"> | string
status?: Prisma.EnumStorageKeyStatusFilter<"StorageKey"> | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFilter<"StorageKey"> | boolean
createdAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
updatedAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
assets?: Prisma.AssetListRelationFilter
@@ -191,6 +199,7 @@ export type StorageKeyOrderByWithRelationInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
isArchived?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
assets?: Prisma.AssetOrderByRelationAggregateInput
@@ -203,6 +212,7 @@ export type StorageKeyWhereUniqueInput = Prisma.AtLeast<{
OR?: Prisma.StorageKeyWhereInput[]
NOT?: Prisma.StorageKeyWhereInput | Prisma.StorageKeyWhereInput[]
status?: Prisma.EnumStorageKeyStatusFilter<"StorageKey"> | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFilter<"StorageKey"> | boolean
createdAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
updatedAt?: Prisma.DateTimeFilter<"StorageKey"> | Date | string
assets?: Prisma.AssetListRelationFilter
@@ -212,6 +222,7 @@ export type StorageKeyOrderByWithAggregationInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
isArchived?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
_count?: Prisma.StorageKeyCountOrderByAggregateInput
@@ -226,6 +237,7 @@ export type StorageKeyScalarWhereWithAggregatesInput = {
id?: Prisma.StringWithAggregatesFilter<"StorageKey"> | string
key?: Prisma.StringWithAggregatesFilter<"StorageKey"> | string
status?: Prisma.EnumStorageKeyStatusWithAggregatesFilter<"StorageKey"> | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolWithAggregatesFilter<"StorageKey"> | boolean
createdAt?: Prisma.DateTimeWithAggregatesFilter<"StorageKey"> | Date | string
updatedAt?: Prisma.DateTimeWithAggregatesFilter<"StorageKey"> | Date | string
}
@@ -234,6 +246,7 @@ export type StorageKeyCreateInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
isArchived?: boolean
createdAt?: Date | string
updatedAt?: Date | string
assets?: Prisma.AssetCreateNestedManyWithoutStorageKeyInput
@@ -243,6 +256,7 @@ export type StorageKeyUncheckedCreateInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
isArchived?: boolean
createdAt?: Date | string
updatedAt?: Date | string
assets?: Prisma.AssetUncheckedCreateNestedManyWithoutStorageKeyInput
@@ -252,6 +266,7 @@ export type StorageKeyUpdateInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFieldUpdateOperationsInput | boolean
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
assets?: Prisma.AssetUpdateManyWithoutStorageKeyNestedInput
@@ -261,6 +276,7 @@ export type StorageKeyUncheckedUpdateInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFieldUpdateOperationsInput | boolean
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
assets?: Prisma.AssetUncheckedUpdateManyWithoutStorageKeyNestedInput
@@ -270,6 +286,7 @@ export type StorageKeyCreateManyInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
isArchived?: boolean
createdAt?: Date | string
updatedAt?: Date | string
}
@@ -278,6 +295,7 @@ export type StorageKeyUpdateManyMutationInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFieldUpdateOperationsInput | boolean
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -286,6 +304,7 @@ export type StorageKeyUncheckedUpdateManyInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFieldUpdateOperationsInput | boolean
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -299,6 +318,7 @@ export type StorageKeyCountOrderByAggregateInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
isArchived?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
}
@@ -307,6 +327,7 @@ export type StorageKeyMaxOrderByAggregateInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
isArchived?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
}
@@ -315,6 +336,7 @@ export type StorageKeyMinOrderByAggregateInput = {
id?: Prisma.SortOrder
key?: Prisma.SortOrder
status?: Prisma.SortOrder
isArchived?: Prisma.SortOrder
createdAt?: Prisma.SortOrder
updatedAt?: Prisma.SortOrder
}
@@ -343,6 +365,7 @@ export type StorageKeyCreateWithoutAssetsInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
isArchived?: boolean
createdAt?: Date | string
updatedAt?: Date | string
}
@@ -351,6 +374,7 @@ export type StorageKeyUncheckedCreateWithoutAssetsInput = {
id?: string
key: string
status?: $Enums.StorageKeyStatus
isArchived?: boolean
createdAt?: Date | string
updatedAt?: Date | string
}
@@ -375,6 +399,7 @@ export type StorageKeyUpdateWithoutAssetsInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFieldUpdateOperationsInput | boolean
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -383,6 +408,7 @@ export type StorageKeyUncheckedUpdateWithoutAssetsInput = {
id?: Prisma.StringFieldUpdateOperationsInput | string
key?: Prisma.StringFieldUpdateOperationsInput | string
status?: Prisma.EnumStorageKeyStatusFieldUpdateOperationsInput | $Enums.StorageKeyStatus
isArchived?: Prisma.BoolFieldUpdateOperationsInput | boolean
createdAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
updatedAt?: Prisma.DateTimeFieldUpdateOperationsInput | Date | string
}
@@ -422,6 +448,7 @@ export type StorageKeySelect<ExtArgs extends runtime.Types.Extensions.InternalAr
id?: boolean
key?: boolean
status?: boolean
isArchived?: boolean
createdAt?: boolean
updatedAt?: boolean
assets?: boolean | Prisma.StorageKey$assetsArgs<ExtArgs>
@@ -432,6 +459,7 @@ export type StorageKeySelectCreateManyAndReturn<ExtArgs extends runtime.Types.Ex
id?: boolean
key?: boolean
status?: boolean
isArchived?: boolean
createdAt?: boolean
updatedAt?: boolean
}, ExtArgs["result"]["storageKey"]>
@@ -440,6 +468,7 @@ export type StorageKeySelectUpdateManyAndReturn<ExtArgs extends runtime.Types.Ex
id?: boolean
key?: boolean
status?: boolean
isArchived?: boolean
createdAt?: boolean
updatedAt?: boolean
}, ExtArgs["result"]["storageKey"]>
@@ -448,11 +477,12 @@ export type StorageKeySelectScalar = {
id?: boolean
key?: boolean
status?: boolean
isArchived?: boolean
createdAt?: boolean
updatedAt?: boolean
}
export type StorageKeyOmit<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetOmit<"id" | "key" | "status" | "createdAt" | "updatedAt", ExtArgs["result"]["storageKey"]>
export type StorageKeyOmit<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = runtime.Types.Extensions.GetOmit<"id" | "key" | "status" | "isArchived" | "createdAt" | "updatedAt", ExtArgs["result"]["storageKey"]>
export type StorageKeyInclude<ExtArgs extends runtime.Types.Extensions.InternalArgs = runtime.Types.Extensions.DefaultArgs> = {
assets?: boolean | Prisma.StorageKey$assetsArgs<ExtArgs>
_count?: boolean | Prisma.StorageKeyCountOutputTypeDefaultArgs<ExtArgs>
@@ -469,6 +499,7 @@ export type $StorageKeyPayload<ExtArgs extends runtime.Types.Extensions.Internal
id: string
key: string
status: $Enums.StorageKeyStatus
isArchived: boolean
createdAt: Date
updatedAt: Date
}, ExtArgs["result"]["storageKey"]>
@@ -898,6 +929,7 @@ export interface StorageKeyFieldRefs {
readonly id: Prisma.FieldRef<"StorageKey", 'String'>
readonly key: Prisma.FieldRef<"StorageKey", 'String'>
readonly status: Prisma.FieldRef<"StorageKey", 'StorageKeyStatus'>
readonly isArchived: Prisma.FieldRef<"StorageKey", 'Boolean'>
readonly createdAt: Prisma.FieldRef<"StorageKey", 'DateTime'>
readonly updatedAt: Prisma.FieldRef<"StorageKey", 'DateTime'>
}
+4
View File
@@ -122,6 +122,7 @@ declare global {
mediaGeneration?: MediaGenerationSettings
appearance?: AppearanceSettings
emailNotification?: EmailNotificationSettings
autoArchive?: import('@shumai/dtos').AutoArchiveSettings
}
// ----------------------------------------------------------------------
@@ -362,6 +363,9 @@ declare global {
/** Render that produced this result, if any (set by the read_asset pipeline). */
assetRead?: AssetReadRequest
watermark?: WatermarkTaskPayload
archive?: {
storageKeyId: string
}
recoveryAttempts?: number
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/dtos",
"version": "0.6.3",
"version": "0.6.4",
"private": true,
"type": "module",
"main": "./src/index.ts",
+45
View File
@@ -158,6 +158,7 @@ export const assetInfoSchema = z.object({
current: z.boolean(),
id: z.string(),
name: z.string().optional().nullable(),
status: z.string().optional().nullable(),
previewUrl: z.string().optional().nullable(),
createdAt: z.string().optional().nullable(),
creator: z
@@ -395,9 +396,53 @@ export type DownloadLinkItem = z.infer<typeof downloadLinkItemSchema>
export const getDownloadLinksResponseSchema = z.object({
files: z.array(downloadLinkItemSchema),
skippedArchivedCount: z.number().optional(),
})
export type GetDownloadLinksResponse = z.infer<typeof getDownloadLinksResponseSchema>
export const getStoragePathsRequestSchema = z.object({
ids: z.array(z.string()).max(100),
})
export type GetStoragePathsRequest = z.infer<typeof getStoragePathsRequestSchema>
export const storagePathItemSchema = z.object({
id: z.string(),
name: z.string(),
path: z.string(),
backend: z.enum(['s3', 'local']),
isArchived: z.boolean(),
})
export type StoragePathItem = z.infer<typeof storagePathItemSchema>
export const getStoragePathsResponseSchema = z.object({
paths: z.array(storagePathItemSchema),
})
export type GetStoragePathsResponse = z.infer<typeof getStoragePathsResponseSchema>
export const archiveAssetsRequestSchema = z.object({
ids: z.array(z.string().min(1)).min(1),
})
export type ArchiveAssetsRequest = z.infer<typeof archiveAssetsRequestSchema>
export const archiveAssetsResponseSchema = z.object({
archivedAssetIds: z.array(z.string()),
taskIds: z.array(z.string()),
skippedCount: z.number(),
})
export type ArchiveAssetsResponse = z.infer<typeof archiveAssetsResponseSchema>
export const restoreArchivedAssetsRequestSchema = z.object({
ids: z.array(z.string().min(1)).min(1),
})
export type RestoreArchivedAssetsRequest = z.infer<typeof restoreArchivedAssetsRequestSchema>
export const restoreArchivedAssetsResponseSchema = z.object({
restoredAssetIds: z.array(z.string()),
taskIds: z.array(z.string()),
skippedCount: z.number(),
})
export type RestoreArchivedAssetsResponse = z.infer<typeof restoreArchivedAssetsResponseSchema>
export const commentExportFormatSchema = z.enum([
'fcp-fiojson',
'media-composer-xml',
+2
View File
@@ -22,6 +22,8 @@ export enum AuditAction {
folder_delete = 'folder_delete',
asset_reparent = 'asset_reparent',
asset_copy = 'asset_copy',
asset_archive = 'asset_archive',
asset_restore_archive = 'asset_restore_archive',
version_stack_create = 'version_stack_create',
version_stack_update = 'version_stack_update',
share_create = 'share_create',
+48
View File
@@ -110,6 +110,32 @@ export type HlsResolutionLadder = (typeof HlsResolutionLadder)[keyof typeof HlsR
export const DEFAULT_HLS_RESOLUTIONS: HlsResolutionLadder[] = ['480p', '720p', '1080p']
export const autoArchiveMethodSchema = z.enum(['not_updated', 'after_created'])
export type AutoArchiveMethod = z.infer<typeof autoArchiveMethodSchema>
export const autoArchiveSettingsSchema = z.object({
enabled: z.boolean(),
method: autoArchiveMethodSchema,
days: z.number().int().min(1).max(3650),
postRestoreGraceDays: z.number().int().min(1).max(3650).default(30),
})
export type AutoArchiveSettings = z.infer<typeof autoArchiveSettingsSchema>
export const DEFAULT_AUTO_ARCHIVE_SETTINGS: AutoArchiveSettings = {
enabled: false,
method: 'not_updated',
days: 30,
postRestoreGraceDays: 30,
}
export interface ArchiveStorageInfo {
enabled: boolean
backend?: 'local' | 's3'
endpoint?: string
bucket?: string
localRoot?: string
}
export const updateTeamSettingsRequestSchema = z.union([
z.object({
key: z.literal('transcode.videoStrategy'),
@@ -139,6 +165,26 @@ export const updateTeamSettingsRequestSchema = z.union([
key: z.literal('appearance.hideAgent'),
value: z.boolean(),
}),
z.object({
key: z.literal('autoArchive'),
value: autoArchiveSettingsSchema,
}),
z.object({
key: z.literal('autoArchive.enabled'),
value: z.boolean(),
}),
z.object({
key: z.literal('autoArchive.method'),
value: autoArchiveMethodSchema,
}),
z.object({
key: z.literal('autoArchive.days'),
value: z.number().int().min(1).max(3650),
}),
z.object({
key: z.literal('autoArchive.postRestoreGraceDays'),
value: z.number().int().min(1).max(3650),
}),
])
export type UpdateTeamSettingsRequest = z.infer<typeof updateTeamSettingsRequestSchema>
@@ -189,6 +235,8 @@ export interface TeamSettingsResponse {
gotenberg?: GotenbergSettingsResponse
appearance?: TeamAppearanceSettings
semanticSearchEnabled?: boolean
archiveStorage?: ArchiveStorageInfo
autoArchive?: AutoArchiveSettings
[key: string]: unknown
}
+1
View File
@@ -32,6 +32,7 @@ export const stackVersionInfoSchema = z.object({
id: z.string(),
version: z.number(),
name: z.string().optional().nullable(),
status: z.string().optional().nullable(),
previewUrl: z.string().optional().nullable(),
createdAt: z.string().optional().nullable(),
creator: z

Some files were not shown because too many files have changed in this diff Show More