mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-20 05:50:09 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0c0b887b8d | ||
|
|
f304caea00 | ||
|
|
65e2ac1383 | ||
|
|
0dc0bbeb03 | ||
|
|
2ecbd49d18 | ||
|
|
ba3c6dbfee | ||
|
|
49a2d7c380 | ||
|
|
6292f3c792 | ||
|
|
20ad0abeba | ||
|
|
fbff404b9b | ||
|
|
c2348b0e68 | ||
|
|
ff3d71ed3d | ||
|
|
77bd1422d8 | ||
|
|
9529c63c0a | ||
|
|
3b00a42a89 |
@@ -12,6 +12,39 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Changed
|
||||
|
||||
## [0.6.4] - 2026-11-05
|
||||
|
||||
### Added
|
||||
|
||||
- **storage**: Add optional archive storage support (`local` folder/NAS or `s3` with configurable storage class and Glacier restore) allowing editors and owners to archive and restore original files while keeping previews, proxy streams, comments, and metadata available in main storage
|
||||
- **storage**: Add team-level automatic asset archival based on age policies (creation time or last modified time) with configurable post-restore grace periods, an hourly background scanning worker, and a dedicated Storage settings page displaying read-only archive backend status and policy configurations
|
||||
|
||||
### Fixed
|
||||
|
||||
- **security**: ⚠️ Several security improvements and access control hardenings. Upgrading to this release is strongly recommended.
|
||||
- **mcp**: Gracefully disable MCP OAuth 2.1 with an error log when deploying on remote HTTP IP addresses without breaking the main server or API token MCP access
|
||||
- **web**: Add immutable cache headers and conditional 304 Not Modified support for production JavaScript and CSS chunks to prevent unnecessary refetches on page reload
|
||||
|
||||
### Changed
|
||||
|
||||
- **file-browser**: Replace the legacy combined "Copy name and download link" action with a context-aware Copy submenu in file browser context menus and card action dropdowns, enabling users to copy image, asset names, in-app links, download links, IDs, Markdown links and storage paths(team-owner only)
|
||||
|
||||
## [0.6.3] - 2026-11-01
|
||||
|
||||
### Added
|
||||
|
||||
- **upload**: Allow users to cancel running upload tasks directly from the upload tasks sidebar with a confirmation dialog, preserving already completed files while discarding incomplete assets
|
||||
- **upload**: Improve upload resilience during network drops, device sleep, and transient gateway errors by adding state-based exponential backoff retries and extending upload link expiration to 12 hours
|
||||
|
||||
### Fixed
|
||||
|
||||
- **upload**: Display a clear diagnostic error toast when remote file uploads fail due to an unconfigured or localhost S3 storage endpoint
|
||||
|
||||
### Changed
|
||||
|
||||
- **upload**: ⚠️ **Important Behavior Change**: Uploading files now appear in their correct sorted position in the file list, following the current sort setting, instead of being pinned to the top and jumping into place after the upload finishes. Cards no longer move when an upload completes. Note that if you sort by name, an uploading file may land outside your current view. Local uploads also show remaining time and transfer speed (e.g. `8m left (390.88 KB/s)`), and other users viewing the folder see `[User] is uploading...`.
|
||||
- **agent**: ⚠️ **Important Behavior Change**: Autofill and Embedding agents will no longer post completion comments when finishing successfully to prevent clutter in asset comment feeds. Comments are now only posted if a task encounters an error.
|
||||
|
||||
## [0.6.2] - 2026-10-29
|
||||
|
||||
### Added
|
||||
|
||||
@@ -30,6 +30,8 @@ RUN bun run scripts/extract-runtime-deps.ts
|
||||
# Stage 2: Runner
|
||||
FROM oven/bun:1.4.4 AS runner
|
||||
|
||||
ENV NODE_ENV=production
|
||||
|
||||
# Install system dependencies and Jellyfin-FFmpeg (with VA-API support)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
|
||||
@@ -25,6 +25,8 @@ RUN bun run scripts/extract-runtime-deps.ts
|
||||
# Stage 2: Runner
|
||||
FROM oven/bun:1.4.4 AS runner
|
||||
|
||||
ENV NODE_ENV=production
|
||||
|
||||
# Install transcoding system dependencies and Jellyfin-FFmpeg (with VA-API/hardware acceleration support)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
Shumai is an open-source Frame.io alternative built for creative teams to collaborate and share media with ease:
|
||||
|
||||
* **S3-Compatible & Local Storage**: Store and serve creative assets securely using the local filesystem or any S3-compatible storage service, including AWS S3, Cloudflare R2, and MinIO.
|
||||
* **Archive Storage & Auto-Archival**: Offload original master media files to secondary local storage or cold S3 storage—manually or automatically via team lifecycle policies—to save on storage costs while keeping proxies for instant browsing and playback. See [Archive Storage documentation](https://docs.shumai.one/configuration/archive-storage).
|
||||
* **Frame-by-Frame Annotations & Comments**: Leave precise feedback with frame-specific drawing tools and timestamped comments on both video and image assets.
|
||||
* **Secure Sharing & Collections**: Create private share links and curated media collections for clients, stakeholders, and collaborators.
|
||||
* **Granular Access Control**: Manage workspace permissions with team-level and project-level role-based access control.
|
||||
@@ -108,7 +109,7 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon
|
||||
> By default, the Docker Compose setups use CPU software transcoding. If your server has a GPU (NVIDIA NVENC, Intel QuickSync, AMD/Intel VA-API, or Rockchip RKMPP) and you want to enable GPU hardware acceleration, check the [Hardware Transcoding Guide](https://docs.shumai.one/configuration/hardware-transcoding).
|
||||
3. Configure environment variables:
|
||||
|
||||
* `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments.
|
||||
* `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments. Note: Remote deployments must use HTTPS for MCP OAuth 2.1; if an HTTP IP address or domain is used, MCP OAuth 2.1 is gracefully disabled while the Web UI and MCP API tokens remain functional.
|
||||
* `AWS_ENDPOINT_URL_S3`: By default, `AWS_ENDPOINT_URL_S3` is set to `http://localhost:{SHUMAI_SERVER_PORT}`. When you use local storage (the default storage backend with `STORAGE_BACKEND: local`) and expose Shumai on a remote server, custom domain, or custom host/port combination, set `AWS_ENDPOINT_URL_S3` to the external URL that client browsers will use to upload and download files.
|
||||
|
||||
For example, if you change the port mapping in `docker-compose.yaml` from `3000:3000` to `12345:3000` and deploy on a server with IP address `12.34.56.78`, set:
|
||||
|
||||
+2
-1
@@ -26,6 +26,7 @@
|
||||
Shumai 是一个专为广告,短剧,漫剧等各类创意团队打造的开源协作管理平台,功能上类似阅流和分秒帧,提供了协作、审片和分享等各种功能:
|
||||
|
||||
* **支持 S3 和本地存储**:你可以把创意素材安全地存放在本地硬盘或者任意兼容 S3 的存储服务里,比如 阿里云 OSS 或 MinIO 等。
|
||||
* **冷归档存储**:支持将原始大文件移至冷存储(如本地挂载的 NAS、AWS S3 Glacier、Google Cloud Storage Coldline 等归档存储类型)以节省主存储成本,同时保留主存储中的预览图、代理流和元数据,不影响网页浏览与即时播放。详情请参阅[归档存储文档](https://docs.shumai.one/configuration/archive-storage)。
|
||||
* **帧级别标注和评论**:支持按帧进行精确反馈,同时支持直接在视频上进行标注,让团队成员能更精确地Get到你的想法。
|
||||
* **安全的分享**:可以生成私密分享链接,同时还支持给分享的图片/视频添加自定义水印,并将水印配置保存为模板以便重复使用。方便给客户、同事和合作方查看。
|
||||
* **细粒度权限控制**:支持按团队、项目设置角色权限,方便管理不同成员的访问范围。
|
||||
@@ -126,7 +127,7 @@ https://gh-proxy.com/https://raw.githubusercontent.com/shumaiOne/shumai/main/REA
|
||||
|
||||
|
||||
3. 配置环境变量:
|
||||
* `BETTER_AUTH_URL`:Shumai 服务的外部公开访问地址(例如 `https://shumai.example.com` 或 `http://12.34.56.78:3000`)。在远程服务器或生产环境中部署时必须配置该变量。
|
||||
* `BETTER_AUTH_URL`:Shumai 服务的外部公开访问地址(例如 `https://shumai.example.com` 或 `http://12.34.56.78:3000`)。在远程服务器或生产环境中部署时必须配置该变量。注意:MCP OAuth 2.1 依赖 HTTPS(本地 loopback 除外);若远程部署使用 HTTP,系统会优雅禁用 MCP OAuth 2.1 并记录日志,Web UI 与基于 API Token 的 MCP 访问不受影响。
|
||||
* `AWS_ENDPOINT_URL_S3`:默认情况下指向 `http://localhost:{SHUMAI_SERVER_PORT}`。当你使用本地存储(即默认配置 `STORAGE_BACKEND: local`)并在远程服务器、自定义域名或自定义端口上公开访问 Shumai 时,请将 `AWS_ENDPOINT_URL_S3` 设置为客户端浏览器上传和下载文件所使用的外部 URL。
|
||||
|
||||
例如,如果你将 `docker-compose.yaml` 中的端口映射从 `3000:3000` 修改为了 `12345:3000`,且服务部署在 IP 为 `12.34.56.78` 的服务器上,请进行如下设置:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/premiere-uxp",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "commonjs",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -90,6 +90,7 @@ apps/web/e2e/
|
||||
| Owner removes version from 2-version stack which dissolves into standalone files | `tests/project/version-stack-manage.spec.ts` |
|
||||
| Dissolving a shared version stack renders the remaining standalone file name on the share page | `tests/project/version-stack-manage.spec.ts` |
|
||||
| Moving a shared file into a shared version stack leaves only 1 stack card on the share page | `tests/project/version-stack-manage.spec.ts` |
|
||||
| Manage versions dialog constrains card width and truncates long file names without dialog overflow | `tests/project/version-stack-manage.spec.ts` |
|
||||
| Owner searches the file list by an English name keyword | `tests/project/search-file-list.spec.ts` |
|
||||
| Owner searches the file list by Chinese, Japanese, and Korean name keywords | `tests/project/search-file-list.spec.ts` |
|
||||
| Owner searches the file list with a keyword and a filter condition | `tests/project/search-file-list.spec.ts` |
|
||||
@@ -97,6 +98,8 @@ apps/web/e2e/
|
||||
| Image share with watermark + download disabled: no download buttons on public list/detail pages, watermarked proxy served, download-url rejected | `tests/project/share-watermark-no-download.spec.ts` |
|
||||
| Video share with watermark + download disabled: no download buttons on public list/detail pages, watermarked proxy served, download-url rejected | `tests/project/share-watermark-no-download.spec.ts` |
|
||||
| Owner views a file and it appears in recents | `tests/project/recents.spec.ts` |
|
||||
| Owner archives a file to archive storage and restores it back to main storage | `tests/project/archive-restore.spec.ts` |
|
||||
| Owner opens Copy submenu in context menu and copies file name | `tests/project/copy-menu.spec.ts` |
|
||||
|
||||
### file
|
||||
| Test | File |
|
||||
|
||||
@@ -10,12 +10,16 @@ export default async function globalTeardown() {
|
||||
if (fs.existsSync(envPath)) {
|
||||
const content = fs.readFileSync(envPath, 'utf8')
|
||||
let bucketName: string | undefined
|
||||
let archiveRoot: string | undefined
|
||||
|
||||
content.split('\n').forEach((line) => {
|
||||
const parts = line.split('=')
|
||||
if (parts.length >= 2 && parts[0].trim() === 'S3_BUCKET') {
|
||||
bucketName = parts.slice(1).join('=').trim()
|
||||
}
|
||||
if (parts.length >= 2 && parts[0].trim() === 'ARCHIVE_LOCAL_ROOT') {
|
||||
archiveRoot = parts.slice(1).join('=').trim()
|
||||
}
|
||||
})
|
||||
|
||||
if (bucketName) {
|
||||
@@ -26,6 +30,11 @@ export default async function globalTeardown() {
|
||||
}
|
||||
}
|
||||
|
||||
if (archiveRoot && fs.existsSync(archiveRoot)) {
|
||||
console.log(`Cleaning up E2E archive storage root: ${archiveRoot}`)
|
||||
fs.rmSync(archiveRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
// Delete the temp .env.e2e file
|
||||
fs.unlinkSync(envPath)
|
||||
}
|
||||
|
||||
@@ -13,8 +13,11 @@ async function start() {
|
||||
// Generate a random S3 bucket name
|
||||
const randomSuffix = randomBytes(4).toString('hex')
|
||||
const bucketName = `shumai-e2e-${randomSuffix}`
|
||||
const archiveRoot = path.resolve('data', `shumai-e2e-archive-${randomSuffix}`)
|
||||
|
||||
process.env.S3_BUCKET = bucketName
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
|
||||
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
|
||||
process.env.SHUMAI_SERVER_PORT = '5200'
|
||||
process.env.BETTER_AUTH_URL = 'http://localhost:5200'
|
||||
process.env.NODE_ENV = 'development'
|
||||
@@ -22,9 +25,12 @@ async function start() {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
|
||||
// Write variables to .env.e2e file so the teardown can load them
|
||||
const envContent = [`DATABASE_URL=${process.env.DATABASE_URL}`, `S3_BUCKET=${bucketName}`].join(
|
||||
'\n',
|
||||
)
|
||||
const envContent = [
|
||||
`DATABASE_URL=${process.env.DATABASE_URL}`,
|
||||
`S3_BUCKET=${bucketName}`,
|
||||
`ARCHIVE_STORAGE_BACKEND=local`,
|
||||
`ARCHIVE_LOCAL_ROOT=${archiveRoot}`,
|
||||
].join('\n')
|
||||
|
||||
fs.writeFileSync(path.resolve(currentDir, '.env.e2e'), envContent)
|
||||
console.log(`E2E env file written. Starting Web App...`)
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { expect, test } from '../../fixtures'
|
||||
import { fileCard } from '../../helpers/files'
|
||||
|
||||
test('owner archives a file to archive storage and restores it back to main storage', async ({
|
||||
file,
|
||||
prisma,
|
||||
}) => {
|
||||
const { page, projectId, fileName, fileId } = file
|
||||
|
||||
await page.goto(`/projects/${projectId}`)
|
||||
const card = fileCard(page, fileName)
|
||||
await expect(card).toBeVisible()
|
||||
|
||||
// 1. Archive via context menu + confirmation dialog
|
||||
await card.click({ button: 'right' })
|
||||
await page.getByTestId('context-menu-archive').click()
|
||||
|
||||
const confirmDialog = page.getByTestId('archive-confirm-dialog')
|
||||
await expect(confirmDialog).toBeVisible()
|
||||
await page.getByTestId('archive-confirm-submit').click()
|
||||
|
||||
// Wait for the DBOS archive workflow to finish and the card badge to settle to Archived
|
||||
await expect(card.getByTestId('file-card-archived-badge')).toBeVisible({ timeout: 15000 })
|
||||
|
||||
const archivedAsset = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: fileId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
expect(archivedAsset.status).toBe('archived')
|
||||
expect(archivedAsset.storageKey?.isArchived).toBe(true)
|
||||
|
||||
// 2. Open detail page and verify Archived header badge + disabled original download button
|
||||
await card.dblclick()
|
||||
await expect(page.getByTestId('detail-header-archived-badge')).toBeVisible()
|
||||
await expect(page.getByTestId('default-viewer-download-button')).toBeDisabled()
|
||||
|
||||
// 3. Return to project file list and restore from archive
|
||||
await page.goto(`/projects/${projectId}`)
|
||||
await expect(card.getByTestId('file-card-archived-badge')).toBeVisible()
|
||||
|
||||
await card.click({ button: 'right' })
|
||||
await page.getByTestId('context-menu-restore-archive').click()
|
||||
|
||||
await expect(confirmDialog).toBeVisible()
|
||||
await page.getByTestId('archive-confirm-submit').click()
|
||||
|
||||
// Wait for the DBOS restore workflow to finish and the Archived badge to disappear
|
||||
await expect(card.getByTestId('file-card-archived-badge')).not.toBeVisible({ timeout: 15000 })
|
||||
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const restored = await prisma.asset.findUnique({
|
||||
where: { id: fileId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
return {
|
||||
status: restored?.status,
|
||||
isArchived: restored?.storageKey?.isArchived,
|
||||
}
|
||||
},
|
||||
{ timeout: 10000 },
|
||||
)
|
||||
.toEqual({ status: 'processed', isArchived: false })
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
import { expect, test } from '../../fixtures'
|
||||
import { fileCard } from '../../helpers/files'
|
||||
|
||||
test('owner sees Copy submenu in context menu and copies file name', async ({
|
||||
file,
|
||||
browserName,
|
||||
}) => {
|
||||
const { page, projectId, fileName } = file
|
||||
|
||||
if (browserName === 'chromium') {
|
||||
await page.context().grantPermissions(['clipboard-read', 'clipboard-write'])
|
||||
}
|
||||
await page.goto(`/projects/${projectId}`)
|
||||
await expect(fileCard(page, fileName)).toBeVisible()
|
||||
|
||||
// Right-click context menu
|
||||
await fileCard(page, fileName).click({ button: 'right' })
|
||||
const copySubTrigger = page.getByRole('menuitem', { name: /^Copy$|^复制$/i })
|
||||
await expect(copySubTrigger).toBeVisible()
|
||||
await copySubTrigger.click()
|
||||
|
||||
// Verify submenu items are visible
|
||||
await expect(page.getByRole('menuitem', { name: /^Name$|^名称$/i })).toBeVisible()
|
||||
await expect(page.getByRole('menuitem', { name: /^Link$|^链接$/i })).toBeVisible()
|
||||
await expect(page.getByRole('menuitem', { name: /^Download link$|^下载链接$/i })).toBeVisible()
|
||||
await expect(page.getByRole('menuitem', { name: /^ID$/i })).toBeVisible()
|
||||
await expect(
|
||||
page.getByRole('menuitem', { name: /^Markdown link$|^Markdown 链接$/i }),
|
||||
).toBeVisible()
|
||||
await expect(page.getByRole('menuitem', { name: /^Storage path$|^存储路径$/i })).toBeVisible()
|
||||
|
||||
// Click "Name"
|
||||
await page.getByRole('menuitem', { name: /^Name$|^名称$/i }).click()
|
||||
const toast = page.locator('[data-sonner-toast]').filter({
|
||||
hasText: /Copied name to clipboard|已复制名称到剪贴板/i,
|
||||
})
|
||||
await expect(toast).toBeVisible()
|
||||
|
||||
if (browserName === 'chromium') {
|
||||
const clipboardText = await page.evaluate(() => navigator.clipboard.readText())
|
||||
expect(clipboardText).toBe(fileName)
|
||||
}
|
||||
})
|
||||
@@ -231,3 +231,76 @@ test('moving a shared file into a shared version stack leaves only 1 stack card
|
||||
const updatedShareRoot = await prisma.asset.findUnique({ where: { id: share.rootFolderId } })
|
||||
expect(updatedShareRoot?.fileCount).toBe(1)
|
||||
})
|
||||
|
||||
test('manage versions dialog constrains card width and truncates long file names without dialog overflow', async ({
|
||||
project,
|
||||
prisma,
|
||||
}) => {
|
||||
const { page, projectId, rootFolderId } = project
|
||||
|
||||
const longName =
|
||||
'super-extremely-long-asset-name-for-testing-manage-versions-dialog-card-overflow-issue-that-keeps-expanding-without-truncating-properly.mp4'
|
||||
const fileA = await seedFile(prisma, projectId, rootFolderId, longName)
|
||||
const fileB = await seedFile(prisma, projectId, rootFolderId, `short-${Date.now()}.mp4`)
|
||||
|
||||
await page.goto(`/projects/${projectId}`)
|
||||
await expect(fileCard(page, fileA.name)).toBeVisible()
|
||||
await expect(fileCard(page, fileB.name)).toBeVisible()
|
||||
|
||||
// Create version stack: drag B on A
|
||||
await dragCard(page, fileCard(page, fileB.name), fileCard(page, fileA.name))
|
||||
await expect(page.getByText('v2', { exact: true })).toBeVisible()
|
||||
|
||||
const stack = await prisma.asset.findFirstOrThrow({
|
||||
where: { projectId, parentId: rootFolderId, type: 'version_stack' },
|
||||
})
|
||||
|
||||
// Navigate to file viewer for the stack
|
||||
await page.goto(`/projects/${projectId}/files/${stack.id}`)
|
||||
|
||||
// Open breadcrumb menu and click "Manage versions..."
|
||||
await page.getByRole('button', { name: new RegExp(fileB.name) }).click()
|
||||
await page.getByRole('menuitem', { name: 'Versions', exact: true }).hover()
|
||||
await page.getByRole('button', { name: /manage versions$/i }).click()
|
||||
|
||||
const dialog = page.getByRole('dialog')
|
||||
await expect(dialog).toBeVisible()
|
||||
|
||||
const rowA = dialog.locator('div.group', { hasText: longName })
|
||||
await expect(rowA).toBeVisible()
|
||||
|
||||
// Measure the dialog and version row
|
||||
const metrics = await page.evaluate(() => {
|
||||
const dialogEl = document.querySelector('[role="dialog"]') as HTMLElement | null
|
||||
if (!dialogEl) return null
|
||||
const row = dialogEl.querySelector('div.group') as HTMLElement | null
|
||||
return {
|
||||
dialogScrollWidth: dialogEl.scrollWidth,
|
||||
dialogClientWidth: dialogEl.clientWidth,
|
||||
dialogOffsetWidth: dialogEl.offsetWidth,
|
||||
rowScrollWidth: row?.scrollWidth,
|
||||
rowClientWidth: row?.clientWidth,
|
||||
}
|
||||
})
|
||||
|
||||
expect(metrics).not.toBeNull()
|
||||
// Dialog should not horizontally scroll or overflow
|
||||
expect(metrics!.dialogScrollWidth).toBeLessThanOrEqual(metrics!.dialogClientWidth)
|
||||
// Row clientWidth should be bounded within the dialog
|
||||
expect(metrics!.rowClientWidth).toBeLessThanOrEqual(metrics!.dialogClientWidth)
|
||||
expect(metrics!.rowScrollWidth).toBeLessThanOrEqual(metrics!.rowClientWidth!)
|
||||
|
||||
// Hover over the row and verify the more-options button is visible and within bounds
|
||||
await rowA.hover()
|
||||
const moreButton = rowA.getByRole('button', { name: 'More options' })
|
||||
await expect(moreButton).toBeVisible()
|
||||
const moreBtnBox = await moreButton.boundingBox()
|
||||
const dialogBox = await dialog.boundingBox()
|
||||
expect(moreBtnBox).not.toBeNull()
|
||||
expect(dialogBox).not.toBeNull()
|
||||
expect(moreBtnBox!.x + moreBtnBox!.width).toBeLessThanOrEqual(dialogBox!.x + dialogBox!.width)
|
||||
|
||||
// Click more options button and verify menu opens
|
||||
await moreButton.click()
|
||||
await expect(page.getByRole('menuitem', { name: /remove from stack/i })).toBeVisible()
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
+30
-15
@@ -8,6 +8,7 @@ import index from '@shumai/webui/index.html'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { app } from '@shumai/api'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { validateArchiveStorageConfig } from '@shumai/core/src/s3/archive-storage'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
@@ -19,6 +20,8 @@ import { authService } from '@shumai/core/src/auth/auth'
|
||||
import { embeddingService } from '@shumai/core/src/search/embedding-service'
|
||||
import { sandboxService } from '@shumai/core'
|
||||
import { notificationJobService } from '@shumai/core/src/notification/notification-job'
|
||||
import { autoArchiveJobService } from '@shumai/core/src/asset/auto-archive-job'
|
||||
import { createHtmlHandler, createStaticFileHandler } from './static'
|
||||
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Web app evaluated successfully!')
|
||||
@@ -76,6 +79,7 @@ if (regenCmdIndex !== -1) {
|
||||
}
|
||||
|
||||
async function run() {
|
||||
validateArchiveStorageConfig()
|
||||
// Initialize workflows and activities for local executor mode
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
@@ -89,6 +93,7 @@ async function run() {
|
||||
await agentService.ensureSystemEmbeddingAgents().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
notificationJobService.start()
|
||||
autoArchiveJobService.start()
|
||||
const args = process.argv.slice(2)
|
||||
let workersOption = process.env.SHUMAI_WORKERS || 'all'
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
@@ -99,13 +104,15 @@ async function run() {
|
||||
}
|
||||
}
|
||||
|
||||
let listenQueues: string[] = ['agent_queue', 'transcode_queue']
|
||||
let listenQueues: string[] = ['agent_queue', 'transcode_queue', 'archive_queue']
|
||||
if (workersOption === 'none') {
|
||||
listenQueues = []
|
||||
} else if (workersOption === 'agent' || workersOption === 'ai') {
|
||||
listenQueues = ['agent_queue']
|
||||
listenQueues = ['agent_queue', 'archive_queue']
|
||||
} else if (workersOption === 'transcode') {
|
||||
listenQueues = ['transcode_queue']
|
||||
} else if (workersOption === 'archive') {
|
||||
listenQueues = ['archive_queue']
|
||||
}
|
||||
|
||||
workflowService.start(listenQueues)
|
||||
@@ -145,7 +152,8 @@ async function run() {
|
||||
for (const file of bundle.files) {
|
||||
const routePath = file.path.startsWith('.') ? file.path.slice(1) : file.path
|
||||
const filePath = join(import.meta.dir, file.path)
|
||||
routes[routePath] = () => new Response(Bun.file(filePath), { headers: file.headers })
|
||||
const isHtml = file.loader === 'html' || file.path.endsWith('.html')
|
||||
routes[routePath] = createStaticFileHandler(filePath, file.headers, !isHtml)
|
||||
}
|
||||
|
||||
const sampleFile = bundle.files[0]?.path || bundle.index || ''
|
||||
@@ -157,13 +165,14 @@ async function run() {
|
||||
const routePath = '/' + pdfWorkerFile
|
||||
const filePath = join(bundleDir, pdfWorkerFile)
|
||||
const bunFile = Bun.file(filePath)
|
||||
routes[routePath] = () =>
|
||||
new Response(bunFile, {
|
||||
headers: {
|
||||
'content-type': 'text/javascript;charset=utf-8',
|
||||
etag: `"${bunFile.size}-${bunFile.lastModified}"`,
|
||||
},
|
||||
})
|
||||
routes[routePath] = createStaticFileHandler(
|
||||
filePath,
|
||||
{
|
||||
'content-type': 'text/javascript;charset=utf-8',
|
||||
etag: `"${bunFile.size}-${bunFile.lastModified}"`,
|
||||
},
|
||||
true,
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Failed to register pdf.worker route:', err)
|
||||
@@ -179,16 +188,22 @@ async function run() {
|
||||
.replaceAll('href="./', 'href="/')
|
||||
.replaceAll('src="./', 'src="/')
|
||||
.replaceAll('content="./', 'content="/')
|
||||
const serveMainHtml = () => new Response(mainHtmlText, { headers: mainHtmlHeaders })
|
||||
const serveMainHtml = createHtmlHandler(mainHtmlText, mainHtmlHeaders)
|
||||
|
||||
routes['/'] = serveMainHtml
|
||||
routes['/*'] = serveMainHtml
|
||||
} else {
|
||||
const fallbackHtmlPath = join(import.meta.dir, 'shumai-app.html')
|
||||
const serveFallbackHtml = () =>
|
||||
new Response(Bun.file(fallbackHtmlPath), {
|
||||
headers: { 'content-type': 'text/html;charset=utf-8' },
|
||||
})
|
||||
const fallbackHtmlFile = Bun.file(fallbackHtmlPath)
|
||||
if (!(await fallbackHtmlFile.exists())) {
|
||||
throw new Error(
|
||||
`Production HTML bundle is missing: neither bundle.files nor ${fallbackHtmlPath} exists.`,
|
||||
)
|
||||
}
|
||||
const fallbackRaw = await fallbackHtmlFile.text()
|
||||
const serveFallbackHtml = createHtmlHandler(fallbackRaw, {
|
||||
'content-type': 'text/html;charset=utf-8',
|
||||
})
|
||||
routes['/'] = serveFallbackHtml
|
||||
routes['/*'] = serveFallbackHtml
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { writeFileSync, unlinkSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { createHtmlHandler, createStaticFileHandler, isEtagMatching, normalizeEtag } from './static'
|
||||
|
||||
describe('Static Asset Serving & Caching', () => {
|
||||
const tempChunkPath = join(tmpdir(), `test-chunk-${Date.now()}.js`)
|
||||
const chunkContent = 'console.log("chunk loaded");'
|
||||
|
||||
beforeAll(() => {
|
||||
writeFileSync(tempChunkPath, chunkContent, 'utf8')
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
try {
|
||||
unlinkSync(tempChunkPath)
|
||||
} catch {
|
||||
// Ignore cleanup error if already deleted
|
||||
}
|
||||
})
|
||||
|
||||
describe('normalizeEtag', () => {
|
||||
it('quotes unquoted ETags according to RFC 9110', () => {
|
||||
expect(normalizeEtag('hHRmXFeFVSk')).toBe('"hHRmXFeFVSk"')
|
||||
expect(normalizeEtag('ACJrxcts9Iw')).toBe('"ACJrxcts9Iw"')
|
||||
})
|
||||
|
||||
it('preserves already-quoted strong and weak ETags', () => {
|
||||
expect(normalizeEtag('"hHRmXFeFVSk"')).toBe('"hHRmXFeFVSk"')
|
||||
expect(normalizeEtag('W/"hHRmXFeFVSk"')).toBe('W/"hHRmXFeFVSk"')
|
||||
})
|
||||
|
||||
it('returns undefined for empty or missing ETags', () => {
|
||||
expect(normalizeEtag(undefined)).toBeUndefined()
|
||||
expect(normalizeEtag('')).toBeUndefined()
|
||||
expect(normalizeEtag(' ')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('isEtagMatching', () => {
|
||||
it('matches exact etag string', () => {
|
||||
expect(isEtagMatching('"12345"', '"12345"')).toBe(true)
|
||||
})
|
||||
|
||||
it('matches weak etag to strong or weak candidates', () => {
|
||||
expect(isEtagMatching('W/"12345"', '"12345"')).toBe(true)
|
||||
expect(isEtagMatching('"12345"', 'W/"12345"')).toBe(true)
|
||||
expect(isEtagMatching('W/"12345"', 'W/"12345"')).toBe(true)
|
||||
})
|
||||
|
||||
it('matches wildcard *', () => {
|
||||
expect(isEtagMatching('"12345"', '*')).toBe(true)
|
||||
})
|
||||
|
||||
it('matches comma-separated candidate lists', () => {
|
||||
expect(isEtagMatching('"12345"', '"old-etag", "12345", "other"')).toBe(true)
|
||||
expect(isEtagMatching('"12345"', '"old-etag", "mismatch"')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns false when etag or ifNoneMatch is missing or empty', () => {
|
||||
expect(isEtagMatching(undefined, '"12345"')).toBe(false)
|
||||
expect(isEtagMatching('"12345"', null)).toBe(false)
|
||||
expect(isEtagMatching('', '"12345"')).toBe(false)
|
||||
expect(isEtagMatching('"12345"', '')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('createStaticFileHandler', () => {
|
||||
it('sets immutable Cache-Control for immutable content-hashed assets', async () => {
|
||||
const handler = createStaticFileHandler(
|
||||
tempChunkPath,
|
||||
{ 'content-type': 'text/javascript;charset=utf-8', etag: '"chunk-hash-1"' },
|
||||
true,
|
||||
)
|
||||
|
||||
const req = new Request('http://localhost/chunk-123.js')
|
||||
const res = handler(req)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('content-type')).toBe('text/javascript;charset=utf-8')
|
||||
expect(res.headers.get('cache-control')).toBe('public, max-age=31536000, immutable')
|
||||
expect(res.headers.get('etag')).toBe('"chunk-hash-1"')
|
||||
expect(await res.text()).toBe(chunkContent)
|
||||
})
|
||||
|
||||
it('returns 304 Not Modified when If-None-Match matches the file etag', async () => {
|
||||
const handler = createStaticFileHandler(
|
||||
tempChunkPath,
|
||||
{ 'content-type': 'text/javascript;charset=utf-8', etag: '"chunk-hash-1"' },
|
||||
true,
|
||||
)
|
||||
|
||||
const req = new Request('http://localhost/chunk-123.js', {
|
||||
headers: { 'if-none-match': '"chunk-hash-1"' },
|
||||
})
|
||||
const res = handler(req)
|
||||
|
||||
expect(res.status).toBe(304)
|
||||
expect(res.headers.get('cache-control')).toBe('public, max-age=31536000, immutable')
|
||||
expect(res.headers.get('etag')).toBe('"chunk-hash-1"')
|
||||
expect(await res.text()).toBe('')
|
||||
})
|
||||
|
||||
it('sets no-cache Cache-Control for non-immutable static assets', async () => {
|
||||
const handler = createStaticFileHandler(
|
||||
tempChunkPath,
|
||||
{ 'content-type': 'text/plain', etag: '"plain-1"' },
|
||||
false,
|
||||
)
|
||||
|
||||
const req = new Request('http://localhost/robots.txt')
|
||||
const res = handler(req)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('cache-control')).toBe('no-cache')
|
||||
})
|
||||
|
||||
it('normalizes unquoted manifest ETags to quoted RFC 9110 strings', async () => {
|
||||
const handler = createStaticFileHandler(
|
||||
tempChunkPath,
|
||||
{ 'content-type': 'text/javascript;charset=utf-8', etag: 'hHRmXFeFVSk' },
|
||||
true,
|
||||
)
|
||||
|
||||
const req = new Request('http://localhost/chunk-123.js')
|
||||
const res = handler(req)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('etag')).toBe('"hHRmXFeFVSk"')
|
||||
|
||||
const revalidateRes = handler(
|
||||
new Request('http://localhost/chunk-123.js', {
|
||||
headers: { 'if-none-match': '"hHRmXFeFVSk"' },
|
||||
}),
|
||||
)
|
||||
expect(revalidateRes.status).toBe(304)
|
||||
expect(revalidateRes.headers.get('etag')).toBe('"hHRmXFeFVSk"')
|
||||
})
|
||||
})
|
||||
|
||||
describe('createHtmlHandler', () => {
|
||||
const htmlText = '<!DOCTYPE html><html><head><title>Shumai</title></head><body></body></html>'
|
||||
|
||||
it('sets no-cache and generates deterministic ETag for HTML', async () => {
|
||||
const handler = createHtmlHandler(htmlText, { 'content-type': 'text/html;charset=utf-8' })
|
||||
|
||||
const req = new Request('http://localhost/')
|
||||
const res = handler(req)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('content-type')).toBe('text/html;charset=utf-8')
|
||||
expect(res.headers.get('cache-control')).toBe('no-cache')
|
||||
const etag = res.headers.get('etag')
|
||||
expect(etag).toBeTruthy()
|
||||
expect(await res.text()).toBe(htmlText)
|
||||
})
|
||||
|
||||
it('returns 304 Not Modified when HTML If-None-Match matches', async () => {
|
||||
const handler = createHtmlHandler(htmlText, { 'content-type': 'text/html;charset=utf-8' })
|
||||
|
||||
const initialRes = handler(new Request('http://localhost/'))
|
||||
const etag = initialRes.headers.get('etag')!
|
||||
|
||||
const revalidateReq = new Request('http://localhost/', {
|
||||
headers: { 'if-none-match': etag },
|
||||
})
|
||||
const revalidateRes = handler(revalidateReq)
|
||||
|
||||
expect(revalidateRes.status).toBe(304)
|
||||
expect(revalidateRes.headers.get('cache-control')).toBe('no-cache')
|
||||
expect(await revalidateRes.text()).toBe('')
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Helpers for serving static production assets with caching headers and ETag revalidation.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Checks whether an incoming `If-None-Match` header matches the asset's ETag.
|
||||
* Supports weak ETags (`W/"..."`), exact matches, and comma-separated lists.
|
||||
*/
|
||||
export function isEtagMatching(etag: string | undefined, ifNoneMatch: string | null): boolean {
|
||||
if (!etag || !ifNoneMatch) {
|
||||
return false
|
||||
}
|
||||
|
||||
const cleanTarget = etag.trim().replace(/^W\//, '').replace(/^"|"$/g, '')
|
||||
if (!cleanTarget) {
|
||||
return false
|
||||
}
|
||||
|
||||
const candidates = ifNoneMatch
|
||||
.split(',')
|
||||
.map((candidate) => candidate.trim().replace(/^W\//, '').replace(/^"|"$/g, ''))
|
||||
|
||||
return candidates.includes(cleanTarget) || candidates.includes('*')
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes an ETag to ensure it is enclosed in quotes according to RFC 9110.
|
||||
* Preserves weak validators (`W/"..."`).
|
||||
*/
|
||||
export function normalizeEtag(etag: string | undefined): string | undefined {
|
||||
if (!etag) {
|
||||
return undefined
|
||||
}
|
||||
const trimmed = etag.trim()
|
||||
if (!trimmed) {
|
||||
return undefined
|
||||
}
|
||||
if (trimmed.startsWith('W/"') && trimmed.endsWith('"')) {
|
||||
return trimmed
|
||||
}
|
||||
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
||||
return trimmed
|
||||
}
|
||||
return `"${trimmed.replace(/^"|"$/g, '')}"`
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a route handler for a static file on disk.
|
||||
* Content-hashed assets (e.g. `chunk-*.js`, `chunk-*.css`) should be marked `isImmutable: true`
|
||||
* to be aggressively cached locally by browsers across page reloads.
|
||||
*/
|
||||
export function createStaticFileHandler(
|
||||
filePath: string,
|
||||
baseHeaders: Record<string, string>,
|
||||
isImmutable: boolean,
|
||||
): (req: Request) => Response {
|
||||
const bunFile = Bun.file(filePath)
|
||||
const rawEtag = baseHeaders.etag || baseHeaders.ETag
|
||||
const normalizedEtag = normalizeEtag(rawEtag)
|
||||
const headers: Record<string, string> = {
|
||||
...baseHeaders,
|
||||
...(normalizedEtag ? { etag: normalizedEtag } : {}),
|
||||
'cache-control': isImmutable ? 'public, max-age=31536000, immutable' : 'no-cache',
|
||||
}
|
||||
const etag = headers.etag
|
||||
|
||||
return (req: Request): Response => {
|
||||
const ifNoneMatch = req.headers.get('if-none-match')
|
||||
if (etag && isEtagMatching(etag, ifNoneMatch)) {
|
||||
return new Response(null, {
|
||||
status: 304,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
return new Response(bunFile, { headers })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a route handler for in-memory transformed HTML entry points.
|
||||
* Sets `cache-control: no-cache` with a deterministic ETag, allowing browsers to cache locally
|
||||
* while always revalidating to ensure new chunk hashes are picked up immediately after deployment.
|
||||
*/
|
||||
export function createHtmlHandler(
|
||||
htmlText: string,
|
||||
baseHeaders: Record<string, string>,
|
||||
): (req: Request) => Response {
|
||||
const rawEtag = baseHeaders.etag || baseHeaders.ETag
|
||||
const etag = normalizeEtag(rawEtag) || `"${Bun.hash(htmlText).toString(16)}"`
|
||||
const headers: Record<string, string> = {
|
||||
...baseHeaders,
|
||||
etag,
|
||||
'cache-control': 'no-cache',
|
||||
}
|
||||
|
||||
return (req: Request): Response => {
|
||||
const ifNoneMatch = req.headers.get('if-none-match')
|
||||
if (isEtagMatching(etag, ifNoneMatch)) {
|
||||
return new Response(null, {
|
||||
status: 304,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
return new Response(htmlText, { headers })
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@
|
||||
},
|
||||
"apps/agent": {
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"bin": {
|
||||
"shumai-agent": "./src/index.ts",
|
||||
},
|
||||
@@ -44,7 +44,7 @@
|
||||
},
|
||||
"apps/cli": {
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"bin": {
|
||||
"shumai-cli": "./src/index.ts",
|
||||
},
|
||||
@@ -61,7 +61,7 @@
|
||||
},
|
||||
"apps/premiere-uxp": {
|
||||
"name": "@shumai/premiere-uxp",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@shumai/api": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
@@ -115,7 +115,7 @@
|
||||
},
|
||||
"apps/transcode": {
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"bin": {
|
||||
"shumai-transcode": "./src/index.ts",
|
||||
},
|
||||
@@ -127,7 +127,7 @@
|
||||
},
|
||||
"apps/web": {
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/api": "workspace:*",
|
||||
@@ -147,7 +147,7 @@
|
||||
},
|
||||
"packages/agent": {
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.74",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
@@ -171,7 +171,7 @@
|
||||
},
|
||||
"packages/api": {
|
||||
"name": "@shumai/api",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@better-auth/mcp": "1.7.12",
|
||||
"@hono/zod-validator": "0.7.6",
|
||||
@@ -191,7 +191,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@shumai/core",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@ai-sdk/alibaba": "2.0.38",
|
||||
"@ai-sdk/amazon-bedrock": "5.0.67",
|
||||
@@ -255,7 +255,7 @@
|
||||
},
|
||||
"packages/db": {
|
||||
"name": "@shumai/db",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@prisma/adapter-pg": "7.10.0",
|
||||
@@ -271,7 +271,7 @@
|
||||
},
|
||||
"packages/dtos": {
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"zod": "4.5.4",
|
||||
@@ -279,7 +279,7 @@
|
||||
},
|
||||
"packages/e2e": {
|
||||
"name": "@shumai/e2e",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@google/genai": "2.28.0",
|
||||
@@ -294,18 +294,18 @@
|
||||
},
|
||||
"packages/operations": {
|
||||
"name": "@shumai/operations",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"zod": "4.5.4",
|
||||
},
|
||||
},
|
||||
"packages/tableprinter": {
|
||||
"name": "@shumai/tableprinter",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
},
|
||||
"packages/transcode": {
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -316,7 +316,7 @@
|
||||
},
|
||||
"packages/webui": {
|
||||
"name": "@shumai/webui",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@base-ui/react": "1.7.0",
|
||||
"@better-auth/oauth-provider": "1.7.12",
|
||||
@@ -419,7 +419,7 @@
|
||||
},
|
||||
"packages/workflow-core": {
|
||||
"name": "@shumai/workflow-core",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"dependencies": {
|
||||
"@dbos-inc/dbos-sdk": "5.2.11",
|
||||
"@shumai/core": "workspace:*",
|
||||
|
||||
@@ -3,6 +3,43 @@ title: "Changelog"
|
||||
description: "Recent updates, new features, and improvements in Shumai"
|
||||
---
|
||||
|
||||
<Update label="v0.6.4" description="November 5, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
- **storage**: Add optional archive storage support (`local` folder/NAS or `s3` with configurable storage class and Glacier restore) allowing editors and owners to archive and restore original files while keeping previews, proxy streams, comments, and metadata available in main storage
|
||||
- **storage**: Add team-level automatic asset archival based on age policies (creation time or last modified time) with configurable post-restore grace periods, an hourly background scanning worker, and a dedicated Storage settings page displaying read-only archive backend status and policy configurations
|
||||
|
||||
### Fixed
|
||||
|
||||
- **security**: ⚠️ Several security improvements and access control hardenings. Upgrading to this release is strongly recommended.
|
||||
- **mcp**: Gracefully disable MCP OAuth 2.1 with an error log when deploying on remote HTTP IP addresses without breaking the main server or API token MCP access
|
||||
- **web**: Add immutable cache headers and conditional 304 Not Modified support for production JavaScript and CSS chunks to prevent unnecessary refetches on page reload
|
||||
|
||||
### Changed
|
||||
|
||||
- **file-browser**: Replace the legacy combined "Copy name and download link" action with a context-aware Copy submenu in file browser context menus and card action dropdowns, enabling users to copy image, asset names, in-app links, download links, IDs, Markdown links and storage paths(team-owner only)
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="v0.6.3" description="November 1, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
- **upload**: Allow users to cancel running upload tasks directly from the upload tasks sidebar with a confirmation dialog, preserving already completed files while discarding incomplete assets
|
||||
- **upload**: Improve upload resilience during network drops, device sleep, and transient gateway errors by adding state-based exponential backoff retries and extending upload link expiration to 12 hours
|
||||
|
||||
### Fixed
|
||||
|
||||
- **upload**: Display a clear diagnostic error toast when remote file uploads fail due to an unconfigured or localhost S3 storage endpoint
|
||||
|
||||
### Changed
|
||||
|
||||
- **upload**: ⚠️ **Important Behavior Change**: Uploading files now appear in their correct sorted position in the file list, following the current sort setting, instead of being pinned to the top and jumping into place after the upload finishes. Cards no longer move when an upload completes. Note that if you sort by name, an uploading file may land outside your current view. Local uploads also show remaining time and transfer speed (e.g. `8m left (390.88 KB/s)`), and other users viewing the folder see `[User] is uploading...`.
|
||||
- **agent**: ⚠️ **Important Behavior Change**: Autofill and Embedding agents will no longer post completion comments when finishing successfully to prevent clutter in asset comment feeds. Comments are now only posted if a task encounters an error.
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="v0.6.2" description="October 29, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
---
|
||||
title: Archive Storage
|
||||
description: Configure secondary cold storage to offload raw media files while preserving in-browser proxy playback and previews.
|
||||
---
|
||||
|
||||
High-resolution camera footage, RAW image sequences, master audio stems, and large 3D assets quickly consume primary storage. Shumai's **Archive Storage** feature enables creative teams to offload heavy master original files to cost-effective cold storage (such as a local NAS, AWS S3 Glacier, or Google Cloud Storage Coldline) while keeping all proxy media, preview images, waveform files, and metadata in primary storage.
|
||||
|
||||
Team members can continue to browse, search, scrub, and review archived assets in the web browser without interruption. When the original raw file is needed for mastering or color grading, it can be restored back to primary storage with a single click.
|
||||
|
||||
---
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Original-Only Archiving**: Only the original raw file is moved to archive storage. Previews, thumbnails, and transcoded proxies (e.g., 180p scrubbing proxies, 1080p streamable MP4s, WebP posters) remain in primary storage.
|
||||
2. **Durable Asynchronous Workflows**: Moving and restoring files is handled by background worker tasks on a dedicated queue (`archive_queue`) managed by DBOS. Tasks automatically survive network interruptions, slow S3 transfers, and server restarts.
|
||||
3. **Transparent Asset Lifecycle**:
|
||||
- When an asset is being archived, its status transitions to `archiving` and settles at `archived`.
|
||||
- When an asset is being restored, its status transitions to `restoring` and returns to `processed` once the original file is back in primary storage.
|
||||
- Trashed assets that are permanently purged automatically delete their original file from archive storage as well.
|
||||
4. **Folder & Version Stack Support**: Archiving a folder or version stack recursively archives all contained file assets. Restoring a folder or version stack restores all archived files inside it.
|
||||
|
||||
---
|
||||
|
||||
## Storage Topologies
|
||||
|
||||
Shumai supports flexible storage configurations where primary storage and archive storage can use different backends or storage providers:
|
||||
|
||||
| Topology | Main Storage | Archive Storage | Description |
|
||||
| :------------------------------------ | :----------- | :-------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Local NAS / Directory** | Local or S3 | `local` | Offload files to a mounted network folder (NFS / SMB) or a secondary cold disk array. |
|
||||
| **Separate S3 Bucket / Provider** | S3 or Local | `s3` | Stream files to a dedicated archive bucket on AWS S3, Wasabi, Backblaze B2, or Google Cloud Storage. |
|
||||
| **Same S3 Bucket (In-Place Tiering)** | AWS S3 | `s3` | Re-tier objects in-place on AWS S3 using cold storage classes (`GLACIER`, `GLACIER_IR`, `DEEP_ARCHIVE`) without re-uploading bytes over the network. |
|
||||
|
||||
---
|
||||
|
||||
## Configuration Variables
|
||||
|
||||
Configure archive storage by adding the following environment variables to your `.env` file or Docker Compose configuration.
|
||||
|
||||
### General Configuration
|
||||
|
||||
| Variable | Description | Required | Default |
|
||||
| :------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :--------------- |
|
||||
| `ARCHIVE_STORAGE_BACKEND` | Storage backend for archives: `local` or `s3`. When unset, archive functionality is disabled in the WebUI. | No | Unset (Disabled) |
|
||||
| `CONCURRENCY_ARCHIVE` | Maximum concurrent archive and restore tasks executed by background workers. | No | `5` |
|
||||
| `SHUMAI_WORKERS` | Worker queues listened to by the application server: `all` (default), `none`, `agent` (or `ai`), `transcode`, `archive`. Set to `all` or `archive` to process tasks. | No | `all` |
|
||||
| `AUTO_ARCHIVE_INTERVAL_MINUTES` | Frequency (in minutes) of the background auto-archive scanner job. Accepts decimal numbers (e.g. `0.02` for testing). | No | `60` (1 hour) |
|
||||
|
||||
### Local Archive Storage
|
||||
|
||||
Used when `ARCHIVE_STORAGE_BACKEND=local`:
|
||||
|
||||
| Variable | Description | Required | Default |
|
||||
| :------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :--------------- |
|
||||
| `ARCHIVE_LOCAL_ROOT` | Host filesystem directory path where archived files are stored. Must not overlap with main local storage directory (`./data` or `./data/<S3_BUCKET>`). | Yes | Unset (Required) |
|
||||
|
||||
### S3 Archive Storage
|
||||
|
||||
Used when `ARCHIVE_STORAGE_BACKEND=s3`. Note that archive storage maintains its own configuration and **never falls back** to the primary `S3_*` environment variables:
|
||||
|
||||
| Variable | Description | Required | Default |
|
||||
| :--------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------- | :------------------------------------ |
|
||||
| `ARCHIVE_S3_BUCKET` | Destination S3 bucket for archived files (can be a dedicated bucket or the same bucket as `S3_BUCKET` for in-place tiering). | Yes | Unset (Required) |
|
||||
| `ARCHIVE_S3_ACCESS_KEY_ID` | Access key ID for the archive S3 service. | Yes | Unset (Required) |
|
||||
| `ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret access key for the archive S3 service. | Yes | Unset (Required) |
|
||||
| `ARCHIVE_AWS_ENDPOINT_URL_S3` | S3 endpoint URL for archive storage (e.g. MinIO, Cloudflare R2, Google Cloud Storage). | No | Unset (AWS standard endpoint) |
|
||||
| `ARCHIVE_S3_REGION` | S3 region for the archive bucket. | No | `auto` |
|
||||
| `ARCHIVE_S3_FORCE_PATH_STYLE` | Use path-style S3 URLs (`true` for MinIO/path-style, `false` for virtual-hosted style). | No | `true` |
|
||||
| `ARCHIVE_S3_STORAGE_CLASS` | S3 storage class to apply to archived objects (e.g. `GLACIER`, `GLACIER_IR`, `DEEP_ARCHIVE`, `COLDLINE`). Required and must not be `STANDARD` when archiving to the same S3 bucket. | No | Unset |
|
||||
| `ARCHIVE_S3_USE_RESTORE_OBJECT` | Controls whether S3 `RestoreObject` is called before reading archived objects (`auto`, `true`, or `false`). | No | `auto` |
|
||||
| `ARCHIVE_S3_RESTORE_DAYS` | Number of days a restored temporary copy remains readable in S3 for asynchronous Glacier restores. | No | `1` |
|
||||
| `ARCHIVE_S3_RESTORE_TIER` | S3 Glacier retrieval tier (`Expedited`, `Standard`, or `Bulk`). Omit when using providers that do not require asynchronous retrieval. | No | Unset |
|
||||
| `ARCHIVE_RESTORE_POLL_INTERVAL_MS` | Milliseconds between status checks while waiting for an asynchronous cold storage restore. | No | `60000` (1 min initial, 5 min steady) |
|
||||
|
||||
---
|
||||
|
||||
## Configuration Examples
|
||||
|
||||
### 1. Local NAS / Mounted Path
|
||||
|
||||
Ideal for self-hosted teams with on-premises Network Attached Storage (Synology, QNAP, TrueNAS, or an NFS/SMB share mounted to the host machine):
|
||||
|
||||
```env
|
||||
# Main storage on local disk or S3
|
||||
STORAGE_BACKEND=local
|
||||
LOCAL_STORAGE_PATH=./storage
|
||||
|
||||
# Archive storage pointing to your mounted NAS directory
|
||||
ARCHIVE_STORAGE_BACKEND=local
|
||||
ARCHIVE_LOCAL_ROOT=/mnt/nas/shumai-archive
|
||||
```
|
||||
|
||||
In `docker-compose.yaml`, mount the host NAS volume into the container:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
shumai:
|
||||
volumes:
|
||||
- /mnt/nas/shumai-archive:/mnt/nas/shumai-archive
|
||||
environment:
|
||||
ARCHIVE_STORAGE_BACKEND: local
|
||||
ARCHIVE_LOCAL_ROOT: /mnt/nas/shumai-archive
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. AWS S3 Glacier Instant Retrieval (Archive)
|
||||
|
||||
AWS S3 Glacier Instant Retrieval (`GLACIER_IR`) offers cold storage pricing with **millisecond retrieval latency**. Because objects in `GLACIER_IR` do not require an asynchronous `RestoreObject` request, restoring an asset to main storage is instantaneous.
|
||||
|
||||
#### Same-Bucket In-Place Tiering (Fast Path)
|
||||
|
||||
If your main storage is AWS S3, you can tier files into `GLACIER_IR` within the **same bucket** without re-uploading bytes over the network:
|
||||
|
||||
```env
|
||||
# Main Storage
|
||||
STORAGE_BACKEND=s3
|
||||
S3_BUCKET=my-shumai-production
|
||||
S3_REGION=us-east-1
|
||||
S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
|
||||
|
||||
# Archive Storage (same bucket & credentials with cold storage class)
|
||||
ARCHIVE_STORAGE_BACKEND=s3
|
||||
ARCHIVE_S3_BUCKET=my-shumai-production
|
||||
ARCHIVE_S3_REGION=us-east-1
|
||||
ARCHIVE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
ARCHIVE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
ARCHIVE_AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
|
||||
ARCHIVE_S3_STORAGE_CLASS=GLACIER_IR
|
||||
```
|
||||
|
||||
#### Dedicated Archive Bucket
|
||||
|
||||
If you prefer keeping archived media in a separate S3 bucket:
|
||||
|
||||
```env
|
||||
# Archive Storage in a separate bucket
|
||||
ARCHIVE_STORAGE_BACKEND=s3
|
||||
ARCHIVE_S3_BUCKET=my-shumai-archive-glacier-ir
|
||||
ARCHIVE_S3_REGION=us-east-1
|
||||
ARCHIVE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
ARCHIVE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
ARCHIVE_AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
|
||||
ARCHIVE_S3_STORAGE_CLASS=GLACIER_IR
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. AWS S3 Glacier Flexible Archive (Cold Storage)
|
||||
|
||||
For maximum cost savings on rarely accessed footage, AWS S3 Flexible Archive (`GLACIER`) or Deep Archive (`DEEP_ARCHIVE`) stores data at the lowest storage rates.
|
||||
|
||||
When a user restores an asset from Glacier, Shumai initiates a `RestoreObject` request, puts the background workflow into durable DBOS sleep, and periodically checks until AWS completes the retrieval:
|
||||
|
||||
```env
|
||||
ARCHIVE_STORAGE_BACKEND=s3
|
||||
ARCHIVE_S3_BUCKET=my-shumai-cold-archive
|
||||
ARCHIVE_S3_REGION=us-east-1
|
||||
ARCHIVE_S3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
ARCHIVE_S3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
ARCHIVE_AWS_ENDPOINT_URL_S3=https://s3.us-east-1.amazonaws.com
|
||||
ARCHIVE_S3_STORAGE_CLASS=GLACIER
|
||||
|
||||
# Keep the restored temporary copy accessible for 7 days
|
||||
ARCHIVE_S3_RESTORE_DAYS=7
|
||||
|
||||
# Glacier retrieval tier: Standard (3-5 hours), Expedited (1-5 min), or Bulk (5-12 hours)
|
||||
ARCHIVE_S3_RESTORE_TIER=Standard
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> During a Glacier restore, DBOS schedules durable sleep timers in PostgreSQL. The worker thread is completely freed and does not consume concurrency slots while waiting for AWS to thaw the file.
|
||||
|
||||
---
|
||||
|
||||
### 4. Google Cloud Storage Coldline (Archive)
|
||||
|
||||
Google Cloud Storage (GCS) provides S3-compatible XML APIs and low-cost storage classes such as `COLDLINE` and `ARCHIVE`. Like Glacier Instant Retrieval, GCS Coldline provides **immediate access** without requiring an asynchronous thaw step:
|
||||
|
||||
```env
|
||||
# Main storage (e.g. Cloudflare R2, MinIO, or standard GCS)
|
||||
STORAGE_BACKEND=s3
|
||||
S3_BUCKET=shumai-main
|
||||
...
|
||||
|
||||
# Archive storage on GCS Coldline
|
||||
ARCHIVE_STORAGE_BACKEND=s3
|
||||
ARCHIVE_S3_BUCKET=shumai-coldline-archive
|
||||
ARCHIVE_S3_ACCESS_KEY_ID=GOOG1EXAMPLETOCKENKEY
|
||||
ARCHIVE_S3_SECRET_ACCESS_KEY=your-gcs-hmac-secret-access-key
|
||||
ARCHIVE_AWS_ENDPOINT_URL_S3=https://storage.googleapis.com
|
||||
ARCHIVE_S3_STORAGE_CLASS=COLDLINE
|
||||
```
|
||||
|
||||
> [!TIP]
|
||||
> Google Cloud Storage objects are readable immediately and do not accept AWS Glacier retrieval parameters. Do not set `ARCHIVE_S3_RESTORE_TIER`. If using GCS's `ARCHIVE` storage class, set `ARCHIVE_S3_USE_RESTORE_OBJECT=false` to bypass S3 `RestoreObject` calls.
|
||||
|
||||
---
|
||||
|
||||
## Automatic Asset Archival
|
||||
|
||||
In addition to manual on-demand archival, Shumai allows teams to automate cold storage tiering based on asset age lifecycle policies. This ensures heavy original master files are automatically offloaded to secondary cold storage without requiring manual housekeeping from creative staff or administrators.
|
||||
|
||||
### WebUI Configuration
|
||||
|
||||
Team owners can configure automatic archival under **Team Settings > Storage**:
|
||||
|
||||
1. **Archive Storage Status Overview**:
|
||||
- Displays the current system archive storage backend (`local` or `s3`), endpoint, bucket, or root directory configured on the server.
|
||||
- If archive storage is not configured in server environment variables, a warning alert is displayed, and auto-archive controls are disabled.
|
||||
- For security, credentials and private server settings are not exposed to client browsers.
|
||||
|
||||
2. **Automatic Asset Archival Policy**:
|
||||
- **Enable Auto Archive**: Toggle automated background archival on or off for the team (disabled by default; requires an active archive storage backend).
|
||||
- **Archive Criteria (`method`)**:
|
||||
- **Not modified for specified days (`not_updated`)**: Evaluates the last modification timestamp (`updatedAt`). Ideal for active production workflows—files stay in fast primary storage as long as team members are actively commenting, annotating, or modifying them. Once no modifications have occurred for the configured duration, the asset is moved to cold storage.
|
||||
- **After creation for specified days (`after_created`)**: Evaluates the initial upload/creation timestamp (`createdAt`). Ideal for strict retention or compliance pipelines where footage is tiered based on its age (e.g., 60 or 90 days after delivery) regardless of recent review comments.
|
||||
- **Age Threshold (`days`)**: The threshold in days (between 1 and 3,650 days) after which eligible assets are queued for archival.
|
||||
- **Post-Restore Grace Period (`postRestoreGraceDays`)**:
|
||||
- When an asset is restored from archive back to primary storage, its modification date is refreshed to the restore time.
|
||||
- When using the **After creation** rule, an asset older than the threshold (e.g. 100 days old) would otherwise be immediately picked up and re-archived by the very next hourly scan.
|
||||
- The post-restore grace period (default: 30 days) guarantees that restored files remain in primary storage for at least the specified number of days before becoming eligible for automated archival again. This value should be set smaller than the Age Threshold. Under this policy, only files that are older than the Age Threshold and have also surpassed the Post-Restore Grace Period since restoration are auto-archived.
|
||||
|
||||
### Sibling Deduplication & Safety
|
||||
|
||||
Shumai uses content-addressable storage keys (`storage_keys`) to deduplicate identical files across projects and folders:
|
||||
|
||||
- **Sibling Protection**: If multiple asset records reference the same underlying storage object (e.g., copied or duplicated files across folders or projects), the auto-archive scanner inspects all sibling assets.
|
||||
- **Strict Consensus**: The underlying storage file is **only** archived if **all active sibling files** satisfy the team's auto-archive criteria and are in a stable `processed` state.
|
||||
- **Starvation-Free Scheduling**: If any sibling copy was modified recently or is currently in an active workflow, the storage key is safely skipped without blocking other eligible assets from being archived.
|
||||
|
||||
### Background Scanner & Execution Semantics
|
||||
|
||||
The auto-archive job runs periodically in the background:
|
||||
|
||||
- **Scan Frequency**: Configured via the `AUTO_ARCHIVE_INTERVAL_MINUTES` environment variable (defaults to `60` minutes; accepts decimal values such as `0.02` for automated testing).
|
||||
- **Distributed Leader Election**: Uses PostgreSQL transaction-scoped advisory locks (`pg_try_advisory_xact_lock`) during the candidate discovery phase. In multi-pod or multi-replica container deployments, only one server instance performs the scan per interval while other replicas cleanly skip.
|
||||
- **Race Condition Prevention**: During archival, candidate assets transition atomically via conditional SQL (`UPDATE assets SET status = 'archiving' WHERE status = 'processed'`) protected by row-level `storage_keys FOR UPDATE` locks. This guarantees zero race conditions with concurrent manual user archival, restoration, or file deletion.
|
||||
|
||||
---
|
||||
|
||||
## WebUI Behavior & Permissions
|
||||
|
||||
- **Permissions**: Only team **Owners** and **Editors** can archive or restore assets.
|
||||
- **Context Menus**: Right-click any file, folder, or version stack in the file browser to select **Archive** or **Restore from Archive**.
|
||||
- **Visual Badges**: Archived items display an `Archived` badge on grid cards and list rows. Inside the file viewer, an archive icon is displayed in front of the file name in the breadcrumb bar.
|
||||
- **Playback & Previews**: Video streaming, scrubbers, waveform audio, PDF rendering, and image viewing remain fully functional using cached proxies.
|
||||
- **Download Restrictions**: The download button for the original raw file is disabled with a tooltip explaining that the raw file is archived. Transcoded proxy downloads (e.g. 1080p MP4) remain available.
|
||||
@@ -38,15 +38,16 @@ Shumai is configured via environment variables. You can define these variables i
|
||||
|
||||
Shumai supports storing files locally or on S3-compatible endpoints.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------ |
|
||||
| `STORAGE_BACKEND` | Storage type. Use `local` for local file system storage, or `s3` for S3-compatible cloud storage. | `local` (default) or `s3` |
|
||||
| `AWS_ENDPOINT_URL_S3` | Endpoint for S3 storage or local storage. Used by the browser to build file upload URLs. Must include port if deploying remotely or using custom ports. | `http://localhost:3000` (respects `SHUMAI_SERVER_PORT` fallback if unset) |
|
||||
| `S3_BUCKET` | S3 bucket name. | `shumai` (default) |
|
||||
| `S3_ACCESS_KEY_ID` | Access key credentials for S3. | `minioadmin` (default) |
|
||||
| `S3_SECRET_ACCESS_KEY` | Secret key credentials for S3. | `minioadmin` (default) |
|
||||
| `S3_REGION` | S3 storage region. | `auto` (default) |
|
||||
| `PRESIGNED_URL_EXPIRES_IN` | Number of hours for which generated presigned download/upload links are valid. | `5` (default) |
|
||||
| Variable | Description | Example / Default |
|
||||
| :-------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------ |
|
||||
| `STORAGE_BACKEND` | Storage type. Use `local` for local file system storage, or `s3` for S3-compatible cloud storage. | `local` (default) or `s3` |
|
||||
| `AWS_ENDPOINT_URL_S3` | Endpoint for S3 storage or local storage. Used by the browser to build file upload URLs. Must include port if deploying remotely or using custom ports. | `http://localhost:3000` (respects `SHUMAI_SERVER_PORT` fallback if unset) |
|
||||
| `S3_BUCKET` | S3 bucket name. | `shumai` (default) |
|
||||
| `S3_ACCESS_KEY_ID` | Access key credentials for S3. | `minioadmin` (default) |
|
||||
| `S3_SECRET_ACCESS_KEY` | Secret key credentials for S3. | `minioadmin` (default) |
|
||||
| `S3_REGION` | S3 storage region. | `auto` (default) |
|
||||
| `PRESIGNED_URL_EXPIRES_IN` | Number of hours for which generated presigned download links are valid. | `5` (default) |
|
||||
| `PRESIGNED_UPLOAD_URL_EXPIRES_IN` | Number of hours for which generated presigned upload links are valid. Falls back to `PRESIGNED_URL_EXPIRES_IN` if unset. | `12` (default) |
|
||||
|
||||
### Local Storage Port Configuration
|
||||
|
||||
@@ -60,6 +61,25 @@ For example, if you deploy Shumai on a remote server at `http://123.456.7.8` and
|
||||
- Set `SHUMAI_SERVER_PORT` to `3000`.
|
||||
- Set `AWS_ENDPOINT_URL_S3` to `http://123.456.7.8:12345`.
|
||||
|
||||
### Archive Storage Config
|
||||
|
||||
Shumai supports configuring an optional **Archive Storage** tier (`local` NAS/directory or `s3` bucket/storage class) for moving original raw files while keeping proxies, previews, thumbnails, and metadata in main storage.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------- |
|
||||
| `ARCHIVE_STORAGE_BACKEND` | Archive storage type. Leave unset to disable archive storage, or set to `local` or `s3`. | Unset (disabled), `local`, or `s3` |
|
||||
| `ARCHIVE_LOCAL_ROOT` | Required when `ARCHIVE_STORAGE_BACKEND=local`. Root directory (e.g., a mounted NAS folder) where archived original files are stored directly under `<ARCHIVE_LOCAL_ROOT>/<key>`. | `/mnt/nas/shumai-archive` |
|
||||
| `ARCHIVE_AWS_ENDPOINT_URL_S3` | Optional custom S3 endpoint for archive storage when `ARCHIVE_STORAGE_BACKEND=s3` (independent of `AWS_ENDPOINT_URL_S3`). | `https://s3.us-east-1.amazonaws.com` |
|
||||
| `ARCHIVE_S3_BUCKET` | Required when `ARCHIVE_STORAGE_BACKEND=s3`. Target S3 bucket for archived files (can be a dedicated bucket or the same bucket as `S3_BUCKET` with a cold storage class). | `shumai-archive` |
|
||||
| `ARCHIVE_S3_ACCESS_KEY_ID` | Required when `ARCHIVE_STORAGE_BACKEND=s3`. S3 access key ID for archive storage. | `AKIA...` |
|
||||
| `ARCHIVE_S3_SECRET_ACCESS_KEY` | Required when `ARCHIVE_STORAGE_BACKEND=s3`. S3 secret access key for archive storage. | `...` |
|
||||
| `ARCHIVE_S3_REGION` | S3 region for archive storage when `ARCHIVE_STORAGE_BACKEND=s3`. | `auto` (default) |
|
||||
| `ARCHIVE_S3_STORAGE_CLASS` | Optional S3 `StorageClass` header applied when archiving objects (accepts any standard or provider-specific class string; required and must not be `STANDARD` for same-bucket tiering). | `GLACIER_IR`, `GLACIER`, `DEEP_ARCHIVE`, `STANDARD_IA` |
|
||||
| `ARCHIVE_S3_USE_RESTORE_OBJECT` | Controls whether S3 `RestoreObject` is called before reading archived objects (`auto`, `true`, or `false`). | `auto` (default) |
|
||||
| `ARCHIVE_S3_RESTORE_DAYS` | Number of days a restored copy of a cold S3 object remains readable during restoration. | `1` (default) |
|
||||
| `ARCHIVE_S3_RESTORE_TIER` | Optional S3 Glacier retrieval tier (`Expedited`, `Standard`, `Bulk`). Omitted when unset for S3-compatible providers that reject `GlacierJobParameters`. | Unset (default) |
|
||||
| `AUTO_ARCHIVE_INTERVAL_MINUTES` | Frequency (in minutes) of the background auto-archive scanner job. Accepts decimal values (e.g. `0.02` for testing). | `60` (1 hour default) |
|
||||
|
||||
---
|
||||
|
||||
## Workflow Config
|
||||
@@ -68,8 +88,9 @@ Configures background worker execution, concurrency limits, and worker identity
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :---------------------- | :-------------------------------------------------------------------------------------------------------------------------------- | :---------------- |
|
||||
| `SHUMAI_WORKERS` | Queues listened to by the web server. Options: `all` (default), `none` (API/UI only), `agent` (or `ai`), `transcode`. | `all` (default) |
|
||||
| `SHUMAI_WORKERS` | Queues listened to by the web server. Options: `all` (default), `none` (API/UI only), `agent` (or `ai`), `transcode`, `archive`. | `all` (default) |
|
||||
| `CONCURRENCY_TRANSCODE` | Maximum concurrent media transcoding tasks executed by workers. | `1` (default) |
|
||||
| `CONCURRENCY_ARCHIVE` | Maximum concurrent archive and restore tasks executed by workers. | `5` (default) |
|
||||
| `CONCURRENCY_AGENT` | Maximum concurrent AI tasks executed by workers. Set to `1` when running local `embedding-server`. | `5` (default) |
|
||||
| `SHUMAI_WORKER_ID` | Optional explicit instance ID for worker replicas in multi-node deployments (auto-derived from hostname and process ID if unset). | `worker-1` |
|
||||
| `POD_NAME` | Kubernetes pod name, used to provide stable per-pod worker identity across StatefulSet restarts. | `transcode-pod-0` |
|
||||
|
||||
+5
-1
@@ -77,7 +77,11 @@
|
||||
},
|
||||
{
|
||||
"group": "Configuration",
|
||||
"pages": ["configuration/env-variables", "configuration/hardware-transcoding"]
|
||||
"pages": [
|
||||
"configuration/env-variables",
|
||||
"configuration/hardware-transcoding",
|
||||
"configuration/archive-storage"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Distributed Deployment",
|
||||
|
||||
@@ -44,7 +44,7 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon
|
||||
3. **Configure Environment Variables**:
|
||||
Open `docker-compose.yaml` to adjust environment variables for your deployment:
|
||||
|
||||
- `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments.
|
||||
- `BETTER_AUTH_URL`: The public base URL of your Shumai server (e.g., `https://shumai.example.com` or `http://12.34.56.78:3000`). This is required for remote servers and production deployments. Note: Remote deployments must use HTTPS for MCP OAuth 2.1; if an HTTP IP address or domain is used, MCP OAuth 2.1 is gracefully disabled while the Web UI and MCP API tokens remain functional.
|
||||
- `AWS_ENDPOINT_URL_S3`: By default, `AWS_ENDPOINT_URL_S3` is set to `http://localhost:{SHUMAI_SERVER_PORT}`. When you use local storage (the default storage backend with `STORAGE_BACKEND: local`) and expose Shumai on a remote server, custom domain, or custom host/port combination, set `AWS_ENDPOINT_URL_S3` to the external URL that client browsers will use to upload and download files.
|
||||
|
||||
For example, if you change the port mapping in `docker-compose.yaml` from `3000:3000` to `12345:3000` and deploy on a server with IP address `12.34.56.78`, set:
|
||||
|
||||
@@ -13,15 +13,16 @@ Using external S3 storage is highly recommended for production teams. It allows
|
||||
|
||||
To enable S3-compatible storage, update the following environment variables in your `.env` file or Docker Compose configuration:
|
||||
|
||||
| Variable | Description | Required | Default / Example |
|
||||
| :------------------------- | :------------------------------------------------------------------------ | :------- | :----------------------------------- |
|
||||
| `STORAGE_BACKEND` | Set to `s3` to enable S3 compatible storage. | Yes | `s3` |
|
||||
| `S3_BUCKET` | The name of the S3 bucket where files will be stored. | Yes | `shumai-bucket` |
|
||||
| `S3_REGION` | The region of your S3 bucket. | No | `auto` |
|
||||
| `S3_ACCESS_KEY_ID` | The access key ID for your storage service. | Yes | `your-access-key-id` |
|
||||
| `S3_SECRET_ACCESS_KEY` | The secret access key for your storage service. | Yes | `your-secret-access-key` |
|
||||
| `AWS_ENDPOINT_URL_S3` | The endpoint URL of your S3 compatible storage provider. | Yes | `https://s3.us-east-1.amazonaws.com` |
|
||||
| `PRESIGNED_URL_EXPIRES_IN` | Duration in hours before generated presigned download/upload URLs expire. | No | `5` |
|
||||
| Variable | Description | Required | Default / Example |
|
||||
| :-------------------------------- | :------------------------------------------------------------------------------------------------------------------ | :------- | :----------------------------------- |
|
||||
| `STORAGE_BACKEND` | Set to `s3` to enable S3 compatible storage. | Yes | `s3` |
|
||||
| `S3_BUCKET` | The name of the S3 bucket where files will be stored. | Yes | `shumai-bucket` |
|
||||
| `S3_REGION` | The region of your S3 bucket. | No | `auto` |
|
||||
| `S3_ACCESS_KEY_ID` | The access key ID for your storage service. | Yes | `your-access-key-id` |
|
||||
| `S3_SECRET_ACCESS_KEY` | The secret access key for your storage service. | Yes | `your-secret-access-key` |
|
||||
| `AWS_ENDPOINT_URL_S3` | The endpoint URL of your S3 compatible storage provider. | Yes | `https://s3.us-east-1.amazonaws.com` |
|
||||
| `PRESIGNED_URL_EXPIRES_IN` | Duration in hours before generated presigned download URLs expire. | No | `5` |
|
||||
| `PRESIGNED_UPLOAD_URL_EXPIRES_IN` | Duration in hours before generated presigned upload URLs expire. Falls back to `PRESIGNED_URL_EXPIRES_IN` if unset. | No | `12` |
|
||||
|
||||
---
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
import * as piAgent from '../index'
|
||||
import { type AgentHarness, type Session } from '@earendil-works/pi-agent-core'
|
||||
import { DatabaseSessionStorage, type DatabaseSessionMetadata } from '../database-session-storage'
|
||||
import { triggerLocalCancel, runInLocalTaskContext } from '@shumai/workflow-core'
|
||||
import { triggerLocalCancel, runInLocalTaskContext, DBOS, DBOSError } from '@shumai/workflow-core'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
import {
|
||||
@@ -1240,6 +1240,28 @@ describe('Agent Database Activities Integration', () => {
|
||||
const deleted = await prisma.assetComment.findUnique({ where: { id: comment.id } })
|
||||
expect(deleted).toBeNull()
|
||||
})
|
||||
|
||||
it('should create comment even if cancelled with DBOSWorkflowCancelledError (runOnCancel: true)', async () => {
|
||||
const realCancelErr = new DBOSError.DBOSWorkflowCancelledError('test-workflow-id')
|
||||
const isWithinWorkflowSpy = vi.spyOn(DBOS, 'isWithinWorkflow').mockReturnValue(true)
|
||||
const runStepSpy = vi.spyOn(DBOS, 'runStep').mockRejectedValue(realCancelErr)
|
||||
|
||||
const comment = await createCommentActivity({
|
||||
assetId: asset.id,
|
||||
message: 'Autofill failed: workflow cancelled',
|
||||
agentId: 'default',
|
||||
})
|
||||
|
||||
expect(comment.id).toBeDefined()
|
||||
expect(comment.message).toBe('Autofill failed: workflow cancelled')
|
||||
|
||||
const savedComment = await prisma.assetComment.findUnique({ where: { id: comment.id } })
|
||||
expect(savedComment).not.toBeNull()
|
||||
expect(savedComment?.message).toBe('Autofill failed: workflow cancelled')
|
||||
|
||||
isWithinWorkflowSpy.mockRestore()
|
||||
runStepSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
describe('Context Retrievals', () => {
|
||||
|
||||
@@ -969,78 +969,86 @@ export const deleteCommentActivity = defineStep(async function deleteCommentActi
|
||||
})
|
||||
})
|
||||
|
||||
export const createCommentActivity = defineStep(async function createCommentActivity(params: {
|
||||
assetId: string
|
||||
message: string
|
||||
sessionId?: string | null
|
||||
agentId?: string | null
|
||||
replyToId?: string | null
|
||||
}) {
|
||||
if (params.sessionId) {
|
||||
const agentId = params.agentId && params.agentId !== 'default' ? params.agentId : 'default'
|
||||
export const createCommentActivity = defineStep(
|
||||
async function createCommentActivity(params: {
|
||||
assetId: string
|
||||
message: string
|
||||
sessionId?: string | null
|
||||
agentId?: string | null
|
||||
replyToId?: string | null
|
||||
}) {
|
||||
if (params.sessionId) {
|
||||
const agentId = params.agentId && params.agentId !== 'default' ? params.agentId : 'default'
|
||||
|
||||
// Fetch teamId from asset
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: params.assetId },
|
||||
include: { project: true },
|
||||
})
|
||||
if (!asset || !asset.project) {
|
||||
throw ApplicationFailure.create({ message: 'asset or project not found', nonRetryable: true })
|
||||
}
|
||||
const teamId = asset.project.teamId
|
||||
|
||||
// Ensure user exists for agent
|
||||
const userExists = await prisma.user.findUnique({ where: { id: agentId } })
|
||||
if (!userExists) {
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: agentId,
|
||||
name: 'Ai Agent',
|
||||
email: `${agentId}@shumai.ai`,
|
||||
type: 'agent',
|
||||
},
|
||||
// Fetch teamId from asset
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: params.assetId },
|
||||
include: { project: true },
|
||||
})
|
||||
}
|
||||
if (!asset || !asset.project) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'asset or project not found',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
const teamId = asset.project.teamId
|
||||
|
||||
// Ensure agent exists
|
||||
const agentExists = await prisma.agent.findUnique({ where: { id: agentId } })
|
||||
if (!agentExists) {
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentId,
|
||||
teamId,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
// Ensure user exists for agent
|
||||
const userExists = await prisma.user.findUnique({ where: { id: agentId } })
|
||||
if (!userExists) {
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: agentId,
|
||||
name: 'Ai Agent',
|
||||
email: `${agentId}@shumai.ai`,
|
||||
type: 'agent',
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure agent exists
|
||||
const agentExists = await prisma.agent.findUnique({ where: { id: agentId } })
|
||||
if (!agentExists) {
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentId,
|
||||
teamId,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure agent session exists
|
||||
const sessionExists = await prisma.agentSession.findUnique({
|
||||
where: { id: params.sessionId },
|
||||
})
|
||||
if (!sessionExists) {
|
||||
await prisma.agentSession.create({
|
||||
data: {
|
||||
id: params.sessionId,
|
||||
agentId: agentId,
|
||||
cwd: process.cwd(),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure agent session exists
|
||||
const sessionExists = await prisma.agentSession.findUnique({ where: { id: params.sessionId } })
|
||||
if (!sessionExists) {
|
||||
await prisma.agentSession.create({
|
||||
data: {
|
||||
id: params.sessionId,
|
||||
agentId: agentId,
|
||||
cwd: process.cwd(),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: params.assetId,
|
||||
message: params.message,
|
||||
sessionId: params.sessionId ?? null,
|
||||
creatorId: params.agentId && params.agentId !== 'default' ? params.agentId : null,
|
||||
replyToId: params.replyToId,
|
||||
},
|
||||
})
|
||||
})
|
||||
return prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: params.assetId,
|
||||
message: params.message,
|
||||
sessionId: params.sessionId ?? null,
|
||||
creatorId: params.agentId && params.agentId !== 'default' ? params.agentId : null,
|
||||
replyToId: params.replyToId,
|
||||
},
|
||||
})
|
||||
},
|
||||
{ runOnCancel: true },
|
||||
)
|
||||
|
||||
export const updateCommentActivity = defineStep(
|
||||
async function updateCommentActivity(params: {
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { AgentToolResult } from '@earendil-works/pi-agent-core'
|
||||
import { downloadAsset, MAX_DOWNLOAD_ASSETS, type DownloadAssetOutput } from '@shumai/operations'
|
||||
import type { User } from '@shumai/db'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { assertStorageKeyNotArchived } from '@shumai/core/src/asset/asset-archive'
|
||||
import { storageBucket } from '@shumai/core/src/asset-read'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
@@ -54,6 +55,7 @@ async function downloadKey(key: string, userId: string): Promise<AgentToolResult
|
||||
type: ResourceType.Asset,
|
||||
id: owningAssetId,
|
||||
})
|
||||
await assertStorageKeyNotArchived(key)
|
||||
const filename = sanitizeFilename(path.basename(key)) || 'downloaded_file'
|
||||
const saved = await saveToSandbox(key, filename)
|
||||
return {
|
||||
|
||||
@@ -67,6 +67,7 @@ describe('operation-backed agent tools', () => {
|
||||
.map((t) => t.name)
|
||||
.sort(),
|
||||
).toEqual([
|
||||
'archive_assets',
|
||||
'copy_assets',
|
||||
'create_comment',
|
||||
'create_file',
|
||||
@@ -87,6 +88,7 @@ describe('operation-backed agent tools', () => {
|
||||
'remove_version',
|
||||
'rename_asset',
|
||||
'resolve_comment',
|
||||
'restore_archived_assets',
|
||||
'search_assets',
|
||||
'update_asset_metadata',
|
||||
])
|
||||
|
||||
@@ -3,6 +3,7 @@ import { agentAutofillMedia } from './agent-autofill'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import * as workflowCore from '@shumai/workflow-core'
|
||||
import { DBOSError } from '@shumai/workflow-core'
|
||||
import * as agentActivities from '../activities/agent'
|
||||
|
||||
vi.mock('@shumai/workflow-core', async () => {
|
||||
@@ -95,13 +96,9 @@ describe('Agent Autofill Workflow', () => {
|
||||
// Verify asset fetched
|
||||
expect(mockActivities.getAssetActivity).toHaveBeenCalledWith('a1')
|
||||
|
||||
// Verify placeholder comment created
|
||||
expect(mockActivities.createCommentActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
message: '__AUTOFILL__',
|
||||
sessionId: 's1',
|
||||
agentId: 'agent-1',
|
||||
})
|
||||
// Verify no success or placeholder comments created
|
||||
expect(mockActivities.createCommentActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateCommentActivity).not.toHaveBeenCalled()
|
||||
|
||||
// Verify fields and context fetched
|
||||
expect(mockActivities.getProjectAutofillFieldsActivity).toHaveBeenCalledWith('p1')
|
||||
@@ -147,13 +144,6 @@ describe('Agent Autofill Workflow', () => {
|
||||
],
|
||||
})
|
||||
|
||||
// Verify placeholder comment updated
|
||||
expect(mockActivities.updateCommentActivity).toHaveBeenCalledWith({
|
||||
commentId: 'comment-placeholder-id',
|
||||
message: 'Autofill completed successfully.',
|
||||
sessionId: 'session-123',
|
||||
})
|
||||
|
||||
// Verify completed task status
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
@@ -205,11 +195,8 @@ describe('Agent Autofill Workflow', () => {
|
||||
|
||||
await agentAutofillMedia(task)
|
||||
|
||||
expect(mockActivities.updateCommentActivity).toHaveBeenCalledWith({
|
||||
commentId: 'comment-placeholder-id',
|
||||
message: 'Autofill completed: No autofill fields defined in project.',
|
||||
})
|
||||
|
||||
expect(mockActivities.createCommentActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateCommentActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.autofillAiActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateAssetMetadataActivity).not.toHaveBeenCalled()
|
||||
|
||||
@@ -219,7 +206,7 @@ describe('Agent Autofill Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle failures, update placeholder with error, and set status to failed', async () => {
|
||||
it('should handle failures, create comment with error, and set status to failed', async () => {
|
||||
mockActivities.getProjectAutofillFieldsActivity.mockRejectedValue(new Error('DB failure'))
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
@@ -232,10 +219,12 @@ describe('Agent Autofill Workflow', () => {
|
||||
|
||||
await expect(agentAutofillMedia(task)).rejects.toThrow('DB failure')
|
||||
|
||||
// Verify placeholder updated with error
|
||||
expect(mockActivities.updateCommentActivity).toHaveBeenCalledWith({
|
||||
commentId: 'comment-placeholder-id',
|
||||
// Verify error comment created
|
||||
expect(mockActivities.createCommentActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
message: 'Autofill failed: DB failure',
|
||||
sessionId: task.id,
|
||||
agentId: 'default',
|
||||
})
|
||||
|
||||
// Verify task failed status
|
||||
@@ -246,6 +235,33 @@ describe('Agent Autofill Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should post an error comment when workflow is cancelled with DBOSWorkflowCancelledError', async () => {
|
||||
const cancelErr = new DBOSError.DBOSWorkflowCancelledError('wf-cancel')
|
||||
mockActivities.getProjectAutofillFieldsActivity.mockRejectedValue(cancelErr)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_metadata_autofill',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(agentAutofillMedia(task)).rejects.toThrow()
|
||||
|
||||
expect(mockActivities.createCommentActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
message: expect.stringContaining('Autofill failed:'),
|
||||
sessionId: task.id,
|
||||
agentId: 'default',
|
||||
})
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
status: 'failed',
|
||||
output: { error: expect.stringContaining('wf-cancel') },
|
||||
})
|
||||
})
|
||||
|
||||
it('should fail with non-retryable ApplicationFailure when asset or project is not found', async () => {
|
||||
mockActivities.getAssetActivity.mockResolvedValue(null)
|
||||
|
||||
|
||||
@@ -12,12 +12,9 @@ import {
|
||||
getAssetActivity,
|
||||
getProjectAutofillFieldsActivity,
|
||||
updateAssetMetadataActivity,
|
||||
updateCommentActivity,
|
||||
} from '../activities/agent'
|
||||
|
||||
export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
let placeholderCommentId: string | undefined
|
||||
|
||||
try {
|
||||
// Update status to processing
|
||||
await updateTaskStatusActivity({
|
||||
@@ -25,16 +22,6 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
status: 'processing',
|
||||
})
|
||||
|
||||
// 0. Create Placeholder Comment
|
||||
const payload = task.payload
|
||||
const placeholder = await createCommentActivity({
|
||||
assetId: task.assetId,
|
||||
message: '__AUTOFILL__',
|
||||
sessionId: payload?.agent?.sessionId || task.id,
|
||||
agentId: payload?.agent?.agentId || 'default',
|
||||
})
|
||||
placeholderCommentId = placeholder.id
|
||||
|
||||
// 1. Get Asset
|
||||
const asset = await getAssetActivity(task.assetId)
|
||||
if (!asset || !asset.project) {
|
||||
@@ -46,12 +33,6 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
// 2. Get Project Autofill Fields
|
||||
const fields = await getProjectAutofillFieldsActivity(projectId)
|
||||
if (fields.length === 0) {
|
||||
if (placeholderCommentId) {
|
||||
await updateCommentActivity({
|
||||
commentId: placeholderCommentId,
|
||||
message: 'Autofill completed: No autofill fields defined in project.',
|
||||
})
|
||||
}
|
||||
await updateTaskStatusActivity({
|
||||
taskId: task.id,
|
||||
status: 'completed',
|
||||
@@ -124,15 +105,6 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
})
|
||||
}
|
||||
|
||||
// 7. Update Placeholder Comment
|
||||
if (placeholderCommentId) {
|
||||
await updateCommentActivity({
|
||||
commentId: placeholderCommentId,
|
||||
message: 'Autofill completed successfully.',
|
||||
sessionId: aiResult.sessionId,
|
||||
})
|
||||
}
|
||||
|
||||
// Update status to completed
|
||||
await updateTaskStatusActivity({
|
||||
taskId: task.id,
|
||||
@@ -141,16 +113,17 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
} catch (err) {
|
||||
console.error(`AgentAutofillMedia failed for task ${task.id}:`, err)
|
||||
|
||||
// Update placeholder comment with error message
|
||||
if (placeholderCommentId) {
|
||||
try {
|
||||
await updateCommentActivity({
|
||||
commentId: placeholderCommentId,
|
||||
message: `Autofill failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
})
|
||||
} catch (commentErr) {
|
||||
console.error('Failed to update error comment:', commentErr)
|
||||
}
|
||||
// Create error comment
|
||||
try {
|
||||
const payload = task.payload
|
||||
await createCommentActivity({
|
||||
assetId: task.assetId,
|
||||
message: `Autofill failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
sessionId: payload?.agent?.sessionId || task.id,
|
||||
agentId: payload?.agent?.agentId || 'default',
|
||||
})
|
||||
} catch (commentErr) {
|
||||
console.error('Failed to create error comment:', commentErr)
|
||||
}
|
||||
|
||||
// Update status to failed
|
||||
|
||||
@@ -3,6 +3,7 @@ import { agentEmbeddingMedia } from './agent-embedding'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import * as workflowCore from '@shumai/workflow-core'
|
||||
import { DBOSError } from '@shumai/workflow-core'
|
||||
import * as agentActivities from '../activities/agent'
|
||||
import * as aiActivities from '../activities/ai'
|
||||
|
||||
@@ -122,13 +123,9 @@ describe('Agent Embedding Workflow', () => {
|
||||
status: 'processing',
|
||||
})
|
||||
|
||||
// Verify placeholder comment created
|
||||
expect(mockActivities.createCommentActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
message: '__EMBEDDING__',
|
||||
sessionId: 's1',
|
||||
agentId: 'agent-1',
|
||||
})
|
||||
// Verify no comments created on success
|
||||
expect(mockActivities.createCommentActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateCommentActivity).not.toHaveBeenCalled()
|
||||
|
||||
// Verify context fetching
|
||||
expect(mockActivities.getEmbeddingContextActivity).toHaveBeenCalledWith({
|
||||
@@ -157,12 +154,6 @@ describe('Agent Embedding Workflow', () => {
|
||||
model: 'gemini-embedding-2',
|
||||
})
|
||||
|
||||
// Verify placeholder comment updated
|
||||
expect(mockActivities.updateCommentActivity).toHaveBeenCalledWith({
|
||||
commentId: 'comment-placeholder-id',
|
||||
message: 'Embedding completed successfully.',
|
||||
})
|
||||
|
||||
// Verify completed task status
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
@@ -417,7 +408,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle failures, update placeholder with error, and set status to failed', async () => {
|
||||
it('should handle failures, create comment with error, and set status to failed', async () => {
|
||||
mockActivities.generateImageEmbeddingActivity.mockRejectedValue(
|
||||
new Error('AI Service Unavailable'),
|
||||
)
|
||||
@@ -433,11 +424,14 @@ describe('Agent Embedding Workflow', () => {
|
||||
|
||||
await expect(agentEmbeddingMedia(task)).rejects.toThrow('AI Service Unavailable')
|
||||
|
||||
// Verify placeholder updated with error
|
||||
expect(mockActivities.updateCommentActivity).toHaveBeenCalledWith({
|
||||
commentId: 'comment-placeholder-id',
|
||||
// Verify error comment created
|
||||
expect(mockActivities.createCommentActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
message: 'Embedding failed: AI Service Unavailable',
|
||||
sessionId: task.id,
|
||||
agentId: 'default',
|
||||
})
|
||||
expect(mockActivities.updateCommentActivity).not.toHaveBeenCalled()
|
||||
|
||||
// Verify status updated to failed
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
@@ -447,6 +441,34 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should post an error comment when workflow is cancelled with DBOSWorkflowCancelledError', async () => {
|
||||
const cancelErr = new DBOSError.DBOSWorkflowCancelledError('wf-cancel')
|
||||
mockActivities.generateImageEmbeddingActivity.mockRejectedValue(cancelErr)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(agentEmbeddingMedia(task)).rejects.toThrow()
|
||||
|
||||
expect(mockActivities.createCommentActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
message: expect.stringContaining('Embedding failed:'),
|
||||
sessionId: task.id,
|
||||
agentId: 'default',
|
||||
})
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
status: 'failed',
|
||||
output: { error: expect.stringContaining('wf-cancel') },
|
||||
})
|
||||
})
|
||||
|
||||
it('should save completed chunks to DB immediately before subsequent chunk failure', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
updateTaskStatusActivity,
|
||||
updateTaskUsageActivity,
|
||||
} from '@shumai/workflow-core'
|
||||
import { createCommentActivity, updateCommentActivity } from '../activities/agent'
|
||||
import { createCommentActivity } from '../activities/agent'
|
||||
import {
|
||||
deleteAssetEmbeddingsActivity,
|
||||
deleteS3ObjectActivity,
|
||||
@@ -25,8 +25,6 @@ export interface GeneratedEmbedding {
|
||||
export type { ExistingAssetEmbeddingInfo }
|
||||
|
||||
export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
let placeholderCommentId: string | undefined
|
||||
|
||||
try {
|
||||
// Update status to processing
|
||||
await updateTaskStatusActivity({
|
||||
@@ -34,16 +32,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
status: 'processing',
|
||||
})
|
||||
|
||||
// 0. Create Placeholder Comment
|
||||
const payload = task.payload
|
||||
const placeholder = await createCommentActivity({
|
||||
assetId: task.assetId,
|
||||
message: '__EMBEDDING__',
|
||||
sessionId: payload?.agent?.sessionId || task.id,
|
||||
agentId: payload?.agent?.agentId || 'default',
|
||||
})
|
||||
placeholderCommentId = placeholder.id
|
||||
|
||||
if (!task.teamId) {
|
||||
throw ApplicationFailure.create({ message: 'Task has no teamId', nonRetryable: true })
|
||||
}
|
||||
@@ -238,14 +226,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
model: usage.model,
|
||||
})
|
||||
|
||||
// 7. Update Placeholder Comment
|
||||
if (placeholderCommentId) {
|
||||
await updateCommentActivity({
|
||||
commentId: placeholderCommentId,
|
||||
message: 'Embedding completed successfully.',
|
||||
})
|
||||
}
|
||||
|
||||
// Update status to completed
|
||||
await updateTaskStatusActivity({
|
||||
taskId: task.id,
|
||||
@@ -254,16 +234,17 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
} catch (err) {
|
||||
console.error(`AgentEmbeddingMedia failed for task ${task.id}:`, err)
|
||||
|
||||
// Update placeholder comment with error message
|
||||
if (placeholderCommentId) {
|
||||
try {
|
||||
await updateCommentActivity({
|
||||
commentId: placeholderCommentId,
|
||||
message: `Embedding failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
})
|
||||
} catch (commentErr) {
|
||||
console.error('Failed to update error comment:', commentErr)
|
||||
}
|
||||
// Create error comment
|
||||
try {
|
||||
const payload = task.payload
|
||||
await createCommentActivity({
|
||||
assetId: task.assetId,
|
||||
message: `Embedding failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
sessionId: payload?.agent?.sessionId || task.id,
|
||||
agentId: payload?.agent?.agentId || 'default',
|
||||
})
|
||||
} catch (commentErr) {
|
||||
console.error('Failed to create error comment:', commentErr)
|
||||
}
|
||||
|
||||
// Update status to failed
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -28,6 +28,7 @@ describe('authn api', () => {
|
||||
expect(await res.json()).toEqual({
|
||||
initialized: true,
|
||||
demoMode: false,
|
||||
archiveStorageEnabled: false,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,15 +2,19 @@ import { Hono } from 'hono'
|
||||
import { teamService } from '@shumai/core/src/team/team'
|
||||
import { userService } from '@shumai/core/src/user/user'
|
||||
import { auth } from '@shumai/core/src/auth/auth'
|
||||
import { createGuestToken } from '@shumai/core/src/auth/guest-token'
|
||||
import { isArchiveStorageEnabled } from '@shumai/core/src/s3/archive-storage'
|
||||
import { z } from 'zod'
|
||||
import { zValidator } from '@hono/zod-validator'
|
||||
import { ulid } from 'ulid'
|
||||
import { setCookie } from 'hono/cookie'
|
||||
|
||||
const app = new Hono()
|
||||
|
||||
const identifySchema = z.object({
|
||||
username: z.string().min(1),
|
||||
email: z.string().email(),
|
||||
shareId: z.string().min(1).optional(),
|
||||
})
|
||||
|
||||
const route = app
|
||||
@@ -19,6 +23,7 @@ const route = app
|
||||
return c.json({
|
||||
initialized: info.initialized,
|
||||
demoMode: info.demoMode,
|
||||
archiveStorageEnabled: isArchiveStorageEnabled(),
|
||||
})
|
||||
})
|
||||
.get('/me', async (c) => {
|
||||
@@ -28,7 +33,7 @@ const route = app
|
||||
return c.json({ id: session?.user?.id ?? null })
|
||||
})
|
||||
.post('/identify', zValidator('json', identifySchema), async (c) => {
|
||||
const { username, email } = c.req.valid('json')
|
||||
const { username, email, shareId } = c.req.valid('json')
|
||||
|
||||
const dummyEmail = `guest_${ulid()}@guest.local`
|
||||
const user = await userService.createGuestUser({
|
||||
@@ -37,6 +42,17 @@ const route = app
|
||||
guestEmail: email,
|
||||
})
|
||||
|
||||
if (shareId) {
|
||||
const token = createGuestToken({ userId: user.id, shareId })
|
||||
setCookie(c, `shumai_guest_${shareId}`, token, {
|
||||
httpOnly: true,
|
||||
path: '/',
|
||||
sameSite: 'Lax',
|
||||
secure: c.req.url.startsWith('https://') || process.env.NODE_ENV === 'production',
|
||||
maxAge: 30 * 24 * 60 * 60,
|
||||
})
|
||||
}
|
||||
|
||||
return c.json({ id: user.id })
|
||||
})
|
||||
.all('/auth/*', (c) => {
|
||||
|
||||
@@ -3,15 +3,19 @@ import { Hono, type Context, type Next } from 'hono'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import fileRoute from './file'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { authMiddleware } from './middleware/auth'
|
||||
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import {
|
||||
archiveAssetsOp,
|
||||
renameAssetOp,
|
||||
resolveCommentOp,
|
||||
restoreArchivedAssetsOp,
|
||||
runOperation,
|
||||
updateAssetMetadataOp,
|
||||
} from '@shumai/core/src/operations'
|
||||
@@ -72,6 +76,7 @@ describe('file api', () => {
|
||||
vi.mocked(authzService.hasPermission).mockClear()
|
||||
|
||||
vi.spyOn(assetService, 'getAsset').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'listAssetsByIds').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'getAssetContext').mockResolvedValue({ teamId: 'test-team' })
|
||||
vi.spyOn(assetService, 'updateAssetName').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'deleteAssets').mockImplementation(vi.fn())
|
||||
@@ -126,6 +131,78 @@ describe('file api', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /files/batch', () => {
|
||||
it('returns empty array when ids is empty', async () => {
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: [] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual([])
|
||||
expect(assetService.listAssetsByIds).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns asset list for accessible IDs', async () => {
|
||||
const assets = [
|
||||
{ id: 'f1', name: 'file-1.mp4', status: 'processing' },
|
||||
{ id: 'f2', name: 'file-2.mp4', status: 'processed' },
|
||||
]
|
||||
vi.mocked(assetService.listAssetsByIds).mockResolvedValue(assets as unknown as AssetInfo[])
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['f1', 'f2'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual(assets)
|
||||
expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1', 'f2'])
|
||||
})
|
||||
|
||||
it('gracefully omits unauthorized or missing assets without failing the batch', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockImplementation(async (req) => {
|
||||
if (req.id === 'unauthorized-id') {
|
||||
throw new HTTPException(403, { message: 'Permission denied' })
|
||||
}
|
||||
})
|
||||
const accessibleAssets = [{ id: 'f1', name: 'file-1.mp4', status: 'processing' }]
|
||||
vi.mocked(assetService.listAssetsByIds).mockResolvedValue(
|
||||
accessibleAssets as unknown as AssetInfo[],
|
||||
)
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['f1', 'unauthorized-id'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual(accessibleAssets)
|
||||
expect(assetService.listAssetsByIds).toHaveBeenCalledWith(['f1'])
|
||||
})
|
||||
|
||||
it('rejects with 400 when more than 100 IDs are requested', async () => {
|
||||
const tooManyIds = Array.from({ length: 101 }, (_, i) => `id-${i}`)
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: tooManyIds }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
})
|
||||
})
|
||||
|
||||
it('PUT /files/:fileId runs rename_asset', async () => {
|
||||
const summary = { id: 'test-id', name: 'new-name', type: 'file', size: 10, projectId: 'p1' }
|
||||
vi.mocked(runOperation).mockResolvedValue(summary)
|
||||
@@ -743,10 +820,13 @@ describe('file api', () => {
|
||||
.spyOn(assetService, 'getProjectIds')
|
||||
.mockResolvedValue(['project-id-1'])
|
||||
|
||||
const mockGetDownloadLinks = vi.spyOn(assetService, 'getDownloadLinks').mockResolvedValue([
|
||||
{ id: 'file1', name: 'file1.txt', url: 'http://link1' },
|
||||
{ id: 'file2', name: 'file2.txt', url: 'http://link2' },
|
||||
])
|
||||
const mockGetDownloadLinks = vi.spyOn(assetService, 'getDownloadLinks').mockResolvedValue({
|
||||
files: [
|
||||
{ id: 'file1', name: 'file1.txt', url: 'http://link1' },
|
||||
{ id: 'file2', name: 'file2.txt', url: 'http://link2' },
|
||||
],
|
||||
skippedArchivedCount: 0,
|
||||
})
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/download-links', {
|
||||
@@ -792,6 +872,127 @@ describe('file api', () => {
|
||||
mockGetProjectIds.mockRestore()
|
||||
})
|
||||
|
||||
describe('POST /files/storage-paths', () => {
|
||||
it('returns empty paths if ids are empty', async () => {
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/storage-paths', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: [] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual({ paths: [] })
|
||||
})
|
||||
|
||||
it('rejects if assets belong to multiple projects', async () => {
|
||||
const mockGetProjectIds = vi
|
||||
.spyOn(assetService, 'getProjectIds')
|
||||
.mockResolvedValue(['proj-1', 'proj-2'])
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/storage-paths', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['asset-1', 'asset-2'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
const json = await res.json()
|
||||
expect(json.error).toBe('All selected items must belong to the same project')
|
||||
|
||||
mockGetProjectIds.mockRestore()
|
||||
})
|
||||
|
||||
it('denies access if user is not a team owner (admin)', async () => {
|
||||
const mockGetProjectIds = vi
|
||||
.spyOn(assetService, 'getProjectIds')
|
||||
.mockResolvedValue(['proj-1'])
|
||||
const mockGetProjectTeam = vi
|
||||
.spyOn(projectService, 'getProjectTeam')
|
||||
.mockResolvedValue('team-1')
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValueOnce(
|
||||
new HTTPException(403, { message: 'Permission denied' }),
|
||||
)
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/storage-paths', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['asset-1'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user1', name: 'Test User' },
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 'team-1',
|
||||
})
|
||||
|
||||
mockGetProjectIds.mockRestore()
|
||||
mockGetProjectTeam.mockRestore()
|
||||
})
|
||||
|
||||
it('returns storage paths if user has team admin permission', async () => {
|
||||
const mockGetProjectIds = vi
|
||||
.spyOn(assetService, 'getProjectIds')
|
||||
.mockResolvedValue(['proj-1'])
|
||||
const mockGetProjectTeam = vi
|
||||
.spyOn(projectService, 'getProjectTeam')
|
||||
.mockResolvedValue('team-1')
|
||||
const mockGetStoragePaths = vi.spyOn(assetService, 'getStoragePaths').mockResolvedValue({
|
||||
paths: [
|
||||
{
|
||||
id: 'asset-1',
|
||||
name: 'clip.mp4',
|
||||
path: 's3://shumai/files/1/clip.mp4',
|
||||
backend: 's3',
|
||||
isArchived: false,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/storage-paths', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: ['asset-1'] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json.paths).toHaveLength(1)
|
||||
expect(json.paths[0].name).toBe('clip.mp4')
|
||||
expect(json.paths[0].path).toBe('s3://shumai/files/1/clip.mp4')
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user1', name: 'Test User' },
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 'team-1',
|
||||
})
|
||||
expect(assetService.getStoragePaths).toHaveBeenCalledWith(['asset-1'])
|
||||
|
||||
mockGetProjectIds.mockRestore()
|
||||
mockGetProjectTeam.mockRestore()
|
||||
mockGetStoragePaths.mockRestore()
|
||||
})
|
||||
|
||||
it('rejects if more than 100 ids are provided', async () => {
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const ids = Array.from({ length: 101 }, (_, i) => `asset-${i}`)
|
||||
const res = await app.request('/files/storage-paths', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
})
|
||||
})
|
||||
|
||||
describe('GET /files/:fileId/m3u8 routes', () => {
|
||||
it('returns master playlist with correct content-type', async () => {
|
||||
const mockMaster = vi
|
||||
@@ -834,4 +1035,64 @@ describe('file api', () => {
|
||||
mockVariant.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /files/archive and POST /files/restore-archive routes', () => {
|
||||
const validUlid = '01ARZ3NDEKTSV4RRFFQ69G5FAV'
|
||||
|
||||
it('archives assets via archiveAssetsOp', async () => {
|
||||
vi.mocked(runOperation).mockResolvedValueOnce({
|
||||
archivedAssetIds: [validUlid],
|
||||
taskIds: ['task-arch-1'],
|
||||
skippedCount: 0,
|
||||
})
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/archive', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: [validUlid] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual({
|
||||
archivedAssetIds: [validUlid],
|
||||
taskIds: ['task-arch-1'],
|
||||
skippedCount: 0,
|
||||
})
|
||||
expect(runOperation).toHaveBeenCalledWith(
|
||||
archiveAssetsOp,
|
||||
{ userId: 'user1', via: 'api' },
|
||||
{ assetIds: [validUlid] },
|
||||
)
|
||||
})
|
||||
|
||||
it('restores archived assets via restoreArchivedAssetsOp', async () => {
|
||||
vi.mocked(runOperation).mockResolvedValueOnce({
|
||||
restoredAssetIds: [validUlid],
|
||||
taskIds: ['task-rest-1'],
|
||||
skippedCount: 0,
|
||||
})
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/files/restore-archive', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ids: [validUlid] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual({
|
||||
restoredAssetIds: [validUlid],
|
||||
taskIds: ['task-rest-1'],
|
||||
skippedCount: 0,
|
||||
})
|
||||
expect(runOperation).toHaveBeenCalledWith(
|
||||
restoreArchivedAssetsOp,
|
||||
{ userId: 'user1', via: 'api' },
|
||||
{ assetIds: [validUlid] },
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,26 +1,33 @@
|
||||
import { zValidator } from '@hono/zod-validator'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { commentService } from '@shumai/core/src/comment/comment'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import {
|
||||
archiveAssetsOp,
|
||||
renameAssetOp,
|
||||
resolveCommentOp,
|
||||
restoreArchivedAssetsOp,
|
||||
runOperation,
|
||||
updateAssetMetadataOp,
|
||||
} from '@shumai/core/src/operations'
|
||||
import {
|
||||
archiveAssetsRequestSchema,
|
||||
createCommentRequestSchema,
|
||||
deleteFilesRequestSchema,
|
||||
batchGetFilesRequestSchema,
|
||||
paginationParamsSchema,
|
||||
restoreArchivedAssetsRequestSchema,
|
||||
restoreFilesRequestSchema,
|
||||
updateAssetMetadataRequestSchema,
|
||||
updateAssetOrderRequestSchema,
|
||||
updateFileRequestSchema,
|
||||
uploadFileRequestSchema,
|
||||
getDownloadLinksRequestSchema,
|
||||
getStoragePathsRequestSchema,
|
||||
completeCommentRequestSchema,
|
||||
exportCommentsRequestSchema,
|
||||
addCommentReactionRequestSchema,
|
||||
@@ -39,6 +46,40 @@ import { z } from 'zod'
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
|
||||
const route = new Hono<{ Variables: { user: User } }>()
|
||||
.post('/files/batch', zValidator('json', batchGetFilesRequestSchema), async (c) => {
|
||||
const user = c.get('user')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
if (req.ids.length === 0) {
|
||||
return c.json([])
|
||||
}
|
||||
|
||||
const accessibleIds: string[] = []
|
||||
await Promise.all(
|
||||
req.ids.map(async (id) => {
|
||||
try {
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id,
|
||||
})
|
||||
accessibleIds.push(id)
|
||||
} catch {
|
||||
// Permissive: silently ignore missing or unauthorized IDs
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
if (accessibleIds.length === 0) {
|
||||
return c.json([])
|
||||
}
|
||||
|
||||
const accessibleSet = new Set(accessibleIds)
|
||||
const orderedAccessibleIds = req.ids.filter((id) => accessibleSet.has(id))
|
||||
const assets = await assetService.listAssetsByIds(orderedAccessibleIds)
|
||||
return c.json(assets)
|
||||
})
|
||||
.get('/files/:fileId', async (c) => {
|
||||
const fileId = c.req.param('fileId')
|
||||
const user = c.get('user')
|
||||
@@ -308,6 +349,28 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
await assetService.restoreAssets(req.ids)
|
||||
return c.body(null, 204)
|
||||
})
|
||||
.post('/files/archive', zValidator('json', archiveAssetsRequestSchema), async (c) => {
|
||||
const req = c.req.valid('json')
|
||||
const result = await runOperation(
|
||||
archiveAssetsOp,
|
||||
{ userId: c.get('user').id, via: 'api' },
|
||||
{ assetIds: req.ids },
|
||||
)
|
||||
return c.json(result)
|
||||
})
|
||||
.post(
|
||||
'/files/restore-archive',
|
||||
zValidator('json', restoreArchivedAssetsRequestSchema),
|
||||
async (c) => {
|
||||
const req = c.req.valid('json')
|
||||
const result = await runOperation(
|
||||
restoreArchivedAssetsOp,
|
||||
{ userId: c.get('user').id, via: 'api' },
|
||||
{ assetIds: req.ids },
|
||||
)
|
||||
return c.json(result)
|
||||
},
|
||||
)
|
||||
.post('/teams/:teamId/files', zValidator('form', uploadFileRequestSchema), async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const user = c.get('user')
|
||||
@@ -389,8 +452,42 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
})
|
||||
|
||||
// 4. Generate download links
|
||||
const files = await assetService.getDownloadLinks(req.ids)
|
||||
return c.json({ files })
|
||||
const { files, skippedArchivedCount } = await assetService.getDownloadLinks(req.ids)
|
||||
return c.json({
|
||||
files,
|
||||
...(skippedArchivedCount > 0 ? { skippedArchivedCount } : {}),
|
||||
})
|
||||
})
|
||||
.post('/files/storage-paths', zValidator('json', getStoragePathsRequestSchema), async (c) => {
|
||||
const user = c.get('user')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
if (req.ids.length === 0) {
|
||||
return c.json({ paths: [] })
|
||||
}
|
||||
|
||||
const projectIds = await assetService.getProjectIds(req.ids)
|
||||
if (projectIds.length === 0) {
|
||||
return c.json({ paths: [] })
|
||||
}
|
||||
|
||||
if (projectIds.length !== 1) {
|
||||
return c.json({ error: 'All selected items must belong to the same project' }, 400)
|
||||
}
|
||||
|
||||
const projectId = projectIds[0]
|
||||
const teamId = await projectService.getProjectTeam(projectId)
|
||||
|
||||
// Verify user has Admin (owner) permission on the team
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const result = await assetService.getStoragePaths(req.ids)
|
||||
return c.json(result)
|
||||
})
|
||||
.post(
|
||||
'/files/download-url',
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
|
||||
import { Hono } from 'hono'
|
||||
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'
|
||||
import { prisma, type User } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { apiTokenService } from '@shumai/core/src/user/api-token'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
|
||||
describe('MCP server on remote HTTP BETTER_AUTH_URL', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
const originalEnv = process.env.BETTER_AUTH_URL
|
||||
let user: User
|
||||
let token: string
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let app: Hono<any>
|
||||
let isMcpOauthEnabled: boolean
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.BETTER_AUTH_URL = 'http://43.133.100.178:3000'
|
||||
vi.resetModules()
|
||||
|
||||
const authModule = await import('@shumai/core/src/auth/auth')
|
||||
isMcpOauthEnabled = authModule.isMcpOauthEnabled
|
||||
const mcpServerModule = await import('./mcp-server')
|
||||
|
||||
app = new Hono().route('/', mcpServerModule.default)
|
||||
|
||||
user = await prisma.user.create({
|
||||
data: { name: 'Remote HTTP User', email: `remote-http-${Date.now()}@test.com` },
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 'Remote HTTP Team' } })
|
||||
await prisma.teamMember.create({ data: { teamId: team.id, userId: user.id, role: 'owner' } })
|
||||
await projectService.createProject(user, { teamId: team.id, name: 'Remote HTTP Project' })
|
||||
|
||||
const tokenRes = await apiTokenService.createToken(user.id, 'remote-http-test')
|
||||
token = tokenRes.token
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
if (originalEnv !== undefined) {
|
||||
process.env.BETTER_AUTH_URL = originalEnv
|
||||
} else {
|
||||
delete process.env.BETTER_AUTH_URL
|
||||
}
|
||||
vi.resetModules()
|
||||
})
|
||||
|
||||
it('omits MCP OAuth plugin and disables isMcpOauthEnabled', () => {
|
||||
expect(isMcpOauthEnabled).toBe(false)
|
||||
})
|
||||
|
||||
it('returns 503 Service Unavailable for unauthenticated requests to /mcp', async () => {
|
||||
const res = await app.request('/mcp', { method: 'POST' })
|
||||
expect(res.status).toBe(503)
|
||||
const body = await res.json()
|
||||
expect(body).toMatchObject({
|
||||
jsonrpc: '2.0',
|
||||
error: {
|
||||
code: -32000,
|
||||
message: expect.stringContaining(
|
||||
'MCP OAuth 2.1 is disabled on this server because BETTER_AUTH_URL is not HTTPS',
|
||||
),
|
||||
},
|
||||
id: null,
|
||||
})
|
||||
})
|
||||
|
||||
it('allows valid API tokens to access /mcp and execute tools even when OAuth is disabled', async () => {
|
||||
const transport = new StreamableHTTPClientTransport(new URL('http://localhost/mcp'), {
|
||||
fetch: async (input, init) => {
|
||||
const headers = new Headers(init?.headers)
|
||||
headers.set('Authorization', `Bearer ${token}`)
|
||||
return app.request(String(input), {
|
||||
...init,
|
||||
headers,
|
||||
})
|
||||
},
|
||||
})
|
||||
const client = new Client({ name: 'remote-http-client', version: '1.0.0' })
|
||||
await client.connect(transport)
|
||||
|
||||
expect(client.getServerVersion()).toMatchObject({ name: 'shumai' })
|
||||
const { tools } = await client.listTools()
|
||||
expect(tools.length).toBeGreaterThan(0)
|
||||
expect(tools.map((t) => t.name)).toContain('list_projects')
|
||||
|
||||
await client.close()
|
||||
})
|
||||
|
||||
it('rejects invalid API tokens with 401 Unauthorized', async () => {
|
||||
const res = await app.request('/mcp', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: 'Bearer shumai_invalidtoken1234567890',
|
||||
},
|
||||
})
|
||||
expect(res.status).toBe(401)
|
||||
})
|
||||
})
|
||||
@@ -6,7 +6,7 @@ import { teamService } from '@shumai/core/src/team/team'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { createFolderOp, readAssetOp, runOperation } from '@shumai/core/src/operations'
|
||||
import { MAX_INLINE_IMAGE_BYTES, MAX_INLINE_IMAGES } from './mcp-extensions/read-asset'
|
||||
import mcpServerRoute, { MCP_SERVER_INSTRUCTIONS } from './mcp-server'
|
||||
import mcpServerRoute, { disabledOauthHandler, MCP_SERVER_INSTRUCTIONS } from './mcp-server'
|
||||
|
||||
vi.mock('@shumai/core/src/operations', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('@shumai/core/src/operations')>()),
|
||||
@@ -53,6 +53,7 @@ describe('MCP server', () => {
|
||||
const { tools } = await client.listTools()
|
||||
|
||||
expect(tools.map((t) => t.name).sort()).toEqual([
|
||||
'archive_assets',
|
||||
'complete_upload',
|
||||
'copy_assets',
|
||||
'create_comment',
|
||||
@@ -74,6 +75,7 @@ describe('MCP server', () => {
|
||||
'remove_version',
|
||||
'rename_asset',
|
||||
'resolve_comment',
|
||||
'restore_archived_assets',
|
||||
'search_assets',
|
||||
'update_asset_metadata',
|
||||
])
|
||||
@@ -400,4 +402,22 @@ describe('MCP server', () => {
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Disabled OAuth Fallback', () => {
|
||||
it('returns 503 Service Unavailable with standard JSON-RPC error payload', async () => {
|
||||
const res = await disabledOauthHandler()
|
||||
expect(res.status).toBe(503)
|
||||
expect(res.headers.get('content-type')).toContain('application/json')
|
||||
const body = await res.json()
|
||||
expect(body).toEqual({
|
||||
jsonrpc: '2.0',
|
||||
error: {
|
||||
code: -32000,
|
||||
message:
|
||||
'MCP OAuth 2.1 is disabled on this server because BETTER_AUTH_URL is not HTTPS or a loopback address. Please configure HTTPS or authenticate with a Shumai API token.',
|
||||
},
|
||||
id: null,
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { z } from 'zod'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
import { dropNullOptionals } from '@shumai/operations'
|
||||
import { getOperationsForSurface, runOperation, type Operation } from '@shumai/core/src/operations'
|
||||
import { auth, authBaseUrl, mcpResource } from '@shumai/core/src/auth/auth'
|
||||
import { auth, authBaseUrl, isMcpOauthEnabled, mcpResource } from '@shumai/core/src/auth/auth'
|
||||
import { apiTokenService } from '@shumai/core/src/user/api-token'
|
||||
import { mcpExtensions, type McpToolExtension } from './mcp-extensions'
|
||||
import { assertWritableInDemoMode, describeOperationError } from './operation-support'
|
||||
@@ -134,40 +134,60 @@ const mcpHandler = createMcpHandler(({ authInfo }) => {
|
||||
return buildMcpServer(userId)
|
||||
})
|
||||
|
||||
const oauthHandler = requireMcpAuth(
|
||||
auth,
|
||||
(req, claims) => {
|
||||
const userId = claims.sub
|
||||
if (!userId) {
|
||||
throw new Error('OAuth token has no sub claim')
|
||||
}
|
||||
const clientId = (claims.azp as string) ?? (claims.client_id as string) ?? ''
|
||||
const scopes =
|
||||
typeof claims.scope === 'string'
|
||||
? claims.scope.split(' ')
|
||||
: ((claims.scopes as string[]) ?? [])
|
||||
return mcpHandler.fetch(req, {
|
||||
authInfo: {
|
||||
token: '',
|
||||
clientId,
|
||||
scopes,
|
||||
extra: { userId },
|
||||
export const disabledOauthHandler = async (): Promise<Response> => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
error: {
|
||||
code: -32000,
|
||||
message:
|
||||
'MCP OAuth 2.1 is disabled on this server because BETTER_AUTH_URL is not HTTPS or a loopback address. Please configure HTTPS or authenticate with a Shumai API token.',
|
||||
},
|
||||
})
|
||||
},
|
||||
{
|
||||
resource: mcpResource,
|
||||
// In-process verification: avoids HTTP loopback round-trip and works even in test environments
|
||||
// where no HTTP port is bound.
|
||||
jwksUrl: (async () => {
|
||||
const res = await auth.handler(new Request(`${authBaseUrl}/api/auth/jwks`))
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to load JWKS in-process: ${res.statusText}`)
|
||||
}
|
||||
return await res.json()
|
||||
}) as unknown as string,
|
||||
},
|
||||
)
|
||||
id: null,
|
||||
}),
|
||||
{
|
||||
status: 503,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
const oauthHandler = isMcpOauthEnabled
|
||||
? requireMcpAuth(
|
||||
auth,
|
||||
(req, claims) => {
|
||||
const userId = claims.sub
|
||||
if (!userId) {
|
||||
throw new Error('OAuth token has no sub claim')
|
||||
}
|
||||
const clientId = (claims.azp as string) ?? (claims.client_id as string) ?? ''
|
||||
const scopes =
|
||||
typeof claims.scope === 'string'
|
||||
? claims.scope.split(' ')
|
||||
: ((claims.scopes as string[]) ?? [])
|
||||
return mcpHandler.fetch(req, {
|
||||
authInfo: {
|
||||
token: '',
|
||||
clientId,
|
||||
scopes,
|
||||
extra: { userId },
|
||||
},
|
||||
})
|
||||
},
|
||||
{
|
||||
resource: mcpResource,
|
||||
// In-process verification: avoids HTTP loopback round-trip and works even in test environments
|
||||
// where no HTTP port is bound.
|
||||
jwksUrl: (async () => {
|
||||
const res = await auth.handler(new Request(`${authBaseUrl}/api/auth/jwks`))
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to load JWKS in-process: ${res.statusText}`)
|
||||
}
|
||||
return await res.json()
|
||||
}) as unknown as string,
|
||||
},
|
||||
)
|
||||
: disabledOauthHandler
|
||||
|
||||
/** Shumai's MCP server (Streamable HTTP). Mounted at `/mcp` with dual API token and OAuth 2.1 auth. */
|
||||
const route = new Hono<{ Variables: { user: User } }>().all('/mcp', async (c) => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Hono } from 'hono'
|
||||
import type { Context } from 'hono'
|
||||
import { getCookie } from 'hono/cookie'
|
||||
import { zValidator } from '@hono/zod-validator'
|
||||
import { shareService } from '@shumai/core/src/share/share'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
@@ -12,6 +13,7 @@ import type { Prisma as PrismaType } from '@shumai/db'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
import { userService } from '@shumai/core/src/user/user'
|
||||
import { auth } from '@shumai/core/src/auth/auth'
|
||||
import { verifyGuestToken } from '@shumai/core/src/auth/guest-token'
|
||||
import { createCommentRequestSchema } from '@shumai/dtos'
|
||||
import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import {
|
||||
@@ -160,7 +162,7 @@ const route = app
|
||||
const password = c.req.header('x-share-password')
|
||||
try {
|
||||
const shareLink = await shareService.getShareLink(shareId)
|
||||
await shareService.verifyPublicAccess(shareLink.rootFolderId, password)
|
||||
await shareService.verifyPublicAccess(shareId, shareLink.rootFolderId, password)
|
||||
return c.json({
|
||||
id: shareLink.id,
|
||||
name: shareLink.name,
|
||||
@@ -184,7 +186,7 @@ const route = app
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.getShareLink(shareId)
|
||||
await shareService.verifyPublicAccess(shareLink.rootFolderId, password)
|
||||
await shareService.verifyPublicAccess(shareId, shareLink.rootFolderId, password)
|
||||
|
||||
const fields = await metadataService.listProjectFields(null, shareLink.projectId)
|
||||
const fieldVisibility = (shareLink.fieldVisibility || {}) as Record<string, boolean>
|
||||
@@ -202,13 +204,14 @@ const route = app
|
||||
'/shares/:shareId/folders/:folderId/children',
|
||||
zValidator('query', sharedChildrenRequestSchema),
|
||||
async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const folderId = c.req.param('folderId')
|
||||
const password =
|
||||
c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
|
||||
const req = c.req.valid('query')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(folderId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, folderId, password)
|
||||
|
||||
let sort: string | undefined
|
||||
let order: string | undefined
|
||||
@@ -264,11 +267,12 @@ const route = app
|
||||
},
|
||||
)
|
||||
.get('/shares/:shareId/files/:fileId', async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(fileId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
|
||||
let asset = await assetService.getAsset({ assetId: fileId })
|
||||
|
||||
@@ -303,12 +307,13 @@ const route = app
|
||||
'/shares/:shareId/files/:fileId/download-url',
|
||||
zValidator('json', z.object({ key: z.string().min(1) })),
|
||||
async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const password = c.req.header('x-share-password')
|
||||
const { key } = c.req.valid('json')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(fileId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
|
||||
if (!shareLink.allowDownload) {
|
||||
throw new ShareLinkDownloadDisabledError('Download is disabled for this share link')
|
||||
@@ -329,12 +334,13 @@ const route = app
|
||||
'/shares/:shareId/files/:fileId/comments',
|
||||
zValidator('query', paginationParamsSchema),
|
||||
async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const password = c.req.header('x-share-password')
|
||||
const req = c.req.valid('query')
|
||||
|
||||
try {
|
||||
await shareService.verifyPublicAccess(fileId, password)
|
||||
await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
|
||||
const targetFileId = await assetService.resolveTargetAssetId(fileId)
|
||||
let userId: string | undefined
|
||||
@@ -343,6 +349,18 @@ const route = app
|
||||
})
|
||||
if (session?.user) {
|
||||
userId = session.user.id
|
||||
} else {
|
||||
const cookieName = `shumai_guest_${shareId}`
|
||||
const token = getCookie(c, cookieName) || getCookie(c, 'shumai_guest')
|
||||
if (token) {
|
||||
const verified = verifyGuestToken(token, shareId)
|
||||
if (verified) {
|
||||
const guestUser = await userService.getUserById(verified.userId)
|
||||
if (guestUser && guestUser.guestEmail) {
|
||||
userId = guestUser.id
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const res = await assetService.listComments(targetFileId, req, userId)
|
||||
return c.json(res)
|
||||
@@ -358,12 +376,15 @@ const route = app
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const password = c.req.header('x-share-password')
|
||||
const guestUserId = c.req.header('x-guest-user-id')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
if (req.attachmentIds && req.attachmentIds.length > 0) {
|
||||
return c.json({ error: 'Attachments are not allowed on public share comments' }, 400)
|
||||
}
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.getShareLink(shareId)
|
||||
await shareService.verifyPublicAccess(fileId, password)
|
||||
await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
|
||||
const targetFileId = await assetService.resolveTargetAssetId(fileId)
|
||||
|
||||
@@ -377,10 +398,17 @@ const route = app
|
||||
|
||||
if (session?.user) {
|
||||
userId = session.user.id
|
||||
} else if (guestUserId) {
|
||||
const guestUser = await userService.getUserById(guestUserId)
|
||||
if (guestUser) {
|
||||
userId = guestUser.id
|
||||
} else {
|
||||
const cookieName = `shumai_guest_${shareId}`
|
||||
const token = getCookie(c, cookieName) || getCookie(c, 'shumai_guest')
|
||||
if (token) {
|
||||
const verified = verifyGuestToken(token, shareId)
|
||||
if (verified) {
|
||||
const guestUser = await userService.getUserById(verified.userId)
|
||||
if (guestUser && guestUser.guestEmail) {
|
||||
userId = guestUser.id
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,7 +428,7 @@ const route = app
|
||||
|
||||
const notifType = req.replyToId ? 'reply_created' : 'comment_created'
|
||||
|
||||
notificationService.create({
|
||||
await notificationService.create({
|
||||
type: notifType,
|
||||
teamId: teamId,
|
||||
creatorId: userId,
|
||||
@@ -415,11 +443,12 @@ const route = app
|
||||
},
|
||||
)
|
||||
.get('/shares/:shareId/files/:fileId/m3u8', async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(fileId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
|
||||
let playlist = await assetService.getHlsMasterPlaylist({
|
||||
assetId: targetAssetId,
|
||||
@@ -438,11 +467,12 @@ const route = app
|
||||
}
|
||||
})
|
||||
.get('/shares/:shareId/files/:fileId/m3u8/master.m3u8', async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(fileId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
|
||||
let playlist = await assetService.getHlsMasterPlaylist({
|
||||
assetId: targetAssetId,
|
||||
@@ -461,6 +491,7 @@ const route = app
|
||||
}
|
||||
})
|
||||
.get('/shares/:shareId/files/:fileId/m3u8/:resolution', async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
let resolution = c.req.param('resolution')
|
||||
if (resolution.endsWith('.m3u8')) {
|
||||
@@ -469,7 +500,7 @@ const route = app
|
||||
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(fileId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
|
||||
const playlist = await assetService.getHlsVariantPlaylist({
|
||||
assetId: targetAssetId,
|
||||
@@ -483,12 +514,13 @@ const route = app
|
||||
}
|
||||
})
|
||||
.get('/shares/:shareId/files/:fileId/m3u8/:resolution/index.m3u8', async (c) => {
|
||||
const shareId = c.req.param('shareId')
|
||||
const fileId = c.req.param('fileId')
|
||||
const resolution = c.req.param('resolution')
|
||||
const password = c.req.header('x-share-password') || c.req.query('password') || c.req.query('p')
|
||||
|
||||
try {
|
||||
const shareLink = await shareService.verifyPublicAccess(fileId, password)
|
||||
const shareLink = await shareService.verifyPublicAccess(shareId, fileId, password)
|
||||
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
|
||||
const playlist = await assetService.getHlsVariantPlaylist({
|
||||
assetId: targetAssetId,
|
||||
|
||||
+54
-15
@@ -16,11 +16,14 @@ describe('S3 API', () => {
|
||||
vi.resetModules()
|
||||
})
|
||||
|
||||
it('GET /files/* calls serveStatic with correct options', async () => {
|
||||
it('GET /files/* calls serveStatic with correct options when valid signature is provided', async () => {
|
||||
const { signLocalGetUrl } = await import('@shumai/core/src/s3/s3')
|
||||
const { signature, expires } = signLocalGetUrl('b1', 'test.webp')
|
||||
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
|
||||
const res = await app.request('/files/b1/test.webp')
|
||||
const res = await app.request(`/files/b1/test.webp?Signature=${signature}&Expires=${expires}`)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.text()).toBe('static-file')
|
||||
@@ -39,7 +42,40 @@ describe('S3 API', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('GET /files/* returns 403 Forbidden when signature is missing', async () => {
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
|
||||
const res = await app.request('/files/b1/test.webp')
|
||||
expect(res.status).toBe(403)
|
||||
const json = await res.json()
|
||||
expect(json.error).toContain('Forbidden')
|
||||
})
|
||||
|
||||
it('GET /files/* returns 403 Forbidden when signature is invalid', async () => {
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
|
||||
const res = await app.request('/files/b1/test.webp?Signature=invalidsig&Expires=9999999999')
|
||||
expect(res.status).toBe(403)
|
||||
})
|
||||
|
||||
it('GET /files/* returns 403 Forbidden when signature is expired', async () => {
|
||||
const { signLocalGetUrl } = await import('@shumai/core/src/s3/s3')
|
||||
// Signed with -10 seconds (expired)
|
||||
const { signature, expires } = signLocalGetUrl('b1', 'test.webp', -10)
|
||||
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
|
||||
const res = await app.request(`/files/b1/test.webp?Signature=${signature}&Expires=${expires}`)
|
||||
expect(res.status).toBe(403)
|
||||
})
|
||||
|
||||
it('GET /files/* sets Content-Disposition with filename when download=1 & filename query params are present', async () => {
|
||||
const { signLocalGetUrl } = await import('@shumai/core/src/s3/s3')
|
||||
const { signature, expires } = signLocalGetUrl('b1', 'test.webp')
|
||||
|
||||
// We need to capture the onFound callback from serveStatic options
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let onFoundCallback: ((path: string, c: any) => void) | undefined
|
||||
@@ -54,7 +90,9 @@ describe('S3 API', () => {
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
|
||||
await app.request('/files/b1/test.webp?download=1&filename=foo.png')
|
||||
await app.request(
|
||||
`/files/b1/test.webp?Signature=${signature}&Expires=${expires}&download=1&filename=foo.png`,
|
||||
)
|
||||
|
||||
expect(onFoundCallback).toBeDefined()
|
||||
|
||||
@@ -101,10 +139,7 @@ describe('S3 API', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('PUT /files/:bucket/:key calls s3Service.putObject', async () => {
|
||||
const { s3Service: service } = await import('@shumai/core/src/s3/s3')
|
||||
const mockPut = vi.spyOn(service, 'putObject').mockResolvedValue(undefined)
|
||||
|
||||
it('PUT /files/:bucket/:key returns 404', async () => {
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
const body = new TextEncoder().encode('test-data')
|
||||
@@ -117,13 +152,17 @@ describe('S3 API', () => {
|
||||
body,
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.text()).toBe('OK')
|
||||
expect(mockPut).toHaveBeenCalledWith(
|
||||
'b1',
|
||||
'test.webp',
|
||||
expect.any(ReadableStream),
|
||||
body.byteLength,
|
||||
)
|
||||
expect(res.status).toBe(404)
|
||||
})
|
||||
|
||||
it('GET /files/* returns 404 when STORAGE_BACKEND is not local', async () => {
|
||||
const s3Module = await import('@shumai/core/src/s3/s3')
|
||||
vi.spyOn(s3Module, 'getStorageBackend').mockReturnValue('s3')
|
||||
|
||||
const { default: route } = await import('./s3')
|
||||
const app = new Hono().route('/files', route)
|
||||
|
||||
const res = await app.request('/files/b1/test.webp')
|
||||
expect(res.status).toBe(404)
|
||||
})
|
||||
})
|
||||
|
||||
+29
-15
@@ -1,10 +1,38 @@
|
||||
import { Hono } from 'hono'
|
||||
import { buildContentDisposition, s3Service } from '@shumai/core/src/s3/s3'
|
||||
import {
|
||||
buildContentDisposition,
|
||||
getStorageBackend,
|
||||
verifyLocalGetUrlSignature,
|
||||
} from '@shumai/core/src/s3/s3'
|
||||
import { serveStatic } from 'hono/bun'
|
||||
|
||||
const route = new Hono()
|
||||
.use('*', async (c, next) => {
|
||||
if (getStorageBackend() !== 'local') {
|
||||
return c.notFound()
|
||||
}
|
||||
return next()
|
||||
})
|
||||
.get(
|
||||
'/*',
|
||||
async (c, next) => {
|
||||
const fullPath = c.req.path.replace(/^\/files\//, '').replace(/^\//, '')
|
||||
const slashIdx = fullPath.indexOf('/')
|
||||
if (slashIdx === -1) {
|
||||
return c.notFound()
|
||||
}
|
||||
const bucket = fullPath.slice(0, slashIdx)
|
||||
const key = fullPath.slice(slashIdx + 1)
|
||||
|
||||
const signature = c.req.query('Signature')
|
||||
const expires = c.req.query('Expires')
|
||||
|
||||
if (!verifyLocalGetUrlSignature(bucket, key, signature, expires)) {
|
||||
return c.json({ error: 'Forbidden: invalid or expired signature' }, 403)
|
||||
}
|
||||
|
||||
return next()
|
||||
},
|
||||
serveStatic({
|
||||
root: './data',
|
||||
rewriteRequestPath: (path) => path.replace(/^\/files\//, ''),
|
||||
@@ -15,19 +43,5 @@ const route = new Hono()
|
||||
},
|
||||
}),
|
||||
)
|
||||
.put('/:bucket/:key{.+}', async (c) => {
|
||||
const bucket = c.req.param('bucket')
|
||||
const key = c.req.param('key')
|
||||
const contentLength = parseInt(c.req.header('content-length') || '0', 10)
|
||||
|
||||
try {
|
||||
const body = c.req.raw.body ?? (await c.req.arrayBuffer())
|
||||
await s3Service.putObject(bucket, key, body, contentLength)
|
||||
return c.text('OK')
|
||||
} catch (e: unknown) {
|
||||
const error = e as Error
|
||||
return c.text(error.message || 'Internal Server Error', 500)
|
||||
}
|
||||
})
|
||||
|
||||
export default route
|
||||
|
||||
@@ -7,6 +7,9 @@ import { shareService } from '@shumai/core/src/share/share'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { ShareLinkPasswordInvalidError, ShareLinkExpiredError } from '@shumai/core/src/share/errors'
|
||||
import { createGuestToken } from '@shumai/core/src/auth/guest-token'
|
||||
import { userService } from '@shumai/core/src/user/user'
|
||||
import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { watermarkService } from '@shumai/core/src/watermark/watermark'
|
||||
@@ -121,7 +124,7 @@ describe('Share API', () => {
|
||||
expect(body.name).toBe('Public Share')
|
||||
expect(body.hasPassword).toBe(true)
|
||||
expect(body.allowDownload).toBe(false)
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('folder1', 'pass')
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'folder1', 'pass')
|
||||
})
|
||||
|
||||
test('Unauthorized', async () => {
|
||||
@@ -206,7 +209,7 @@ describe('Share API', () => {
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('folder1', 'pass')
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'folder1', 'pass')
|
||||
expect(assetService.listChildren).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
assetId: 'folder1',
|
||||
@@ -989,7 +992,7 @@ describe('Share API', () => {
|
||||
expect(res.status).toBe(200)
|
||||
const body = await res.json()
|
||||
expect(body.url).toBe('https://s3.example.com/download')
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('symlink1', undefined)
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'symlink1', undefined)
|
||||
expect(assetService.getDownloadUrl).toHaveBeenCalledWith('symlink1', 'files/target1/fly.png')
|
||||
})
|
||||
|
||||
@@ -1033,7 +1036,7 @@ describe('Share API', () => {
|
||||
expect(res.headers.get('content-type')).toContain('application/vnd.apple.mpegurl')
|
||||
const text = await res.text()
|
||||
expect(text).toContain('1080p/index.m3u8?p=secret123')
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('file1', 'secret123')
|
||||
expect(shareService.verifyPublicAccess).toHaveBeenCalledWith('share1', 'file1', 'secret123')
|
||||
expect(assetService.getHlsMasterPlaylist).toHaveBeenCalledWith({
|
||||
assetId: 'file1',
|
||||
watermarkConfigId: null,
|
||||
@@ -1093,4 +1096,106 @@ describe('Share API', () => {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /shares/:shareId/files/:fileId/comments', () => {
|
||||
test('rejects comments with attachmentIds on public shares', async () => {
|
||||
const res = await app.request('/shares/share1/files/file1/comments', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
message: 'hi',
|
||||
attachmentIds: ['att1'],
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
const body = await res.json()
|
||||
expect(body.error).toContain('Attachments are not allowed')
|
||||
})
|
||||
|
||||
test('rejects spoofed x-guest-user-id header without cookie', async () => {
|
||||
vi.spyOn(shareService, 'getShareLink').mockResolvedValue({
|
||||
id: 'share1',
|
||||
projectId: 'p1',
|
||||
} as unknown as Awaited<ReturnType<typeof shareService.getShareLink>>)
|
||||
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue({
|
||||
id: 'share1',
|
||||
} as unknown as Awaited<ReturnType<typeof shareService.verifyPublicAccess>>)
|
||||
vi.spyOn(assetService, 'resolveTargetAssetId').mockResolvedValue('file1')
|
||||
|
||||
const res = await app.request('/shares/share1/files/file1/comments', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-guest-user-id': 'victim_user',
|
||||
},
|
||||
body: JSON.stringify({ message: 'spoofed comment', attachmentIds: [] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(401)
|
||||
})
|
||||
|
||||
test('allows valid guest cookie bound to shareId and creates comment', async () => {
|
||||
vi.spyOn(shareService, 'getShareLink').mockResolvedValue({
|
||||
id: 'share1',
|
||||
projectId: 'p1',
|
||||
} as unknown as Awaited<ReturnType<typeof shareService.getShareLink>>)
|
||||
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue({
|
||||
id: 'share1',
|
||||
} as unknown as Awaited<ReturnType<typeof shareService.verifyPublicAccess>>)
|
||||
vi.spyOn(assetService, 'resolveTargetAssetId').mockResolvedValue('file1')
|
||||
vi.spyOn(userService, 'getUserById').mockResolvedValue({
|
||||
id: 'guest1',
|
||||
name: 'Guest Alice',
|
||||
guestEmail: 'alice@guest.com',
|
||||
} as unknown as Awaited<ReturnType<typeof userService.getUserById>>)
|
||||
vi.spyOn(assetService, 'createComment').mockResolvedValue({
|
||||
id: 'comm1',
|
||||
message: 'hello from alice',
|
||||
} as unknown as Awaited<ReturnType<typeof assetService.createComment>>)
|
||||
vi.spyOn(notificationService, 'create').mockResolvedValue(
|
||||
undefined as unknown as Awaited<ReturnType<typeof notificationService.create>>,
|
||||
)
|
||||
|
||||
const token = createGuestToken({ userId: 'guest1', shareId: 'share1' })
|
||||
const res = await app.request('/shares/share1/files/file1/comments', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Cookie: `shumai_guest_share1=${token}`,
|
||||
},
|
||||
body: JSON.stringify({ message: 'hello from alice', attachmentIds: [] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(201)
|
||||
expect(assetService.createComment).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
userId: 'guest1',
|
||||
message: 'hello from alice',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test('rejects guest cookie bound to a different shareId', async () => {
|
||||
vi.spyOn(shareService, 'getShareLink').mockResolvedValue({
|
||||
id: 'share1',
|
||||
projectId: 'p1',
|
||||
} as unknown as Awaited<ReturnType<typeof shareService.getShareLink>>)
|
||||
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue({
|
||||
id: 'share1',
|
||||
} as unknown as Awaited<ReturnType<typeof shareService.verifyPublicAccess>>)
|
||||
|
||||
const wrongToken = createGuestToken({ userId: 'guest1', shareId: 'other-share' })
|
||||
const res = await app.request('/shares/share1/files/file1/comments', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Cookie: `shumai_guest_share1=${wrongToken}`,
|
||||
},
|
||||
body: JSON.stringify({ message: 'should fail', attachmentIds: [] }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(401)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,7 +5,6 @@ import teamRoute from './team'
|
||||
import { authMiddleware } from './middleware/auth'
|
||||
import { teamService } from '@shumai/core/src/team/team'
|
||||
import { userMetadataService } from '@shumai/core/src/user-metadata/user-metadata'
|
||||
import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
@@ -80,14 +79,12 @@ describe('team api', () => {
|
||||
const app = new Hono().use('*', authMiddleware).route('/', teamRoute)
|
||||
let mockCreateTeam: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockGetUserTeams: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockJoinTeam: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockGetMe: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockGetTeamMembers: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockGetSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockUpdateSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockListUserMetadata: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockUpsertUserMetadata: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockCreateNotification: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockGetSandboxSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockUpdateSandboxSettings: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
let mockUpdateMe: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
@@ -95,14 +92,12 @@ describe('team api', () => {
|
||||
beforeEach(() => {
|
||||
mockCreateTeam = vi.spyOn(teamService, 'createTeam')
|
||||
mockGetUserTeams = vi.spyOn(teamService, 'getUserTeams')
|
||||
mockJoinTeam = vi.spyOn(teamService, 'joinTeam')
|
||||
mockGetMe = vi.spyOn(teamService, 'getMe')
|
||||
mockGetTeamMembers = vi.spyOn(teamService, 'getTeamMembers')
|
||||
mockGetSettings = vi.spyOn(teamService, 'getSettings')
|
||||
mockUpdateSettings = vi.spyOn(teamService, 'updateSettings')
|
||||
mockListUserMetadata = vi.spyOn(userMetadataService, 'listMetadata')
|
||||
mockUpsertUserMetadata = vi.spyOn(userMetadataService, 'upsertMetadata')
|
||||
mockCreateNotification = vi.spyOn(notificationService, 'create').mockResolvedValue()
|
||||
mockGetSandboxSettings = vi.spyOn(teamService, 'getSandboxSettings')
|
||||
mockUpdateSandboxSettings = vi.spyOn(teamService, 'updateSandboxSettings')
|
||||
mockUpdateMe = vi.spyOn(teamService, 'updateMe')
|
||||
@@ -148,20 +143,9 @@ describe('team api', () => {
|
||||
expect(mockGetUserTeams).toHaveBeenCalledWith(expect.objectContaining({ userId: 'user1' }))
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/members joins team and sends notification', async () => {
|
||||
mockJoinTeam.mockResolvedValue(undefined)
|
||||
|
||||
it('POST /teams/:teamId/members returns 404', async () => {
|
||||
const res = await app.request('/teams/t1/members', { method: 'POST' })
|
||||
|
||||
expect(res.status).toBe(204)
|
||||
expect(mockJoinTeam).toHaveBeenCalledWith({ teamId: 't1', userId: 'user1' })
|
||||
expect(mockCreateNotification).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: 'new_user_join_team',
|
||||
teamId: 't1',
|
||||
userId: 'user1',
|
||||
}),
|
||||
)
|
||||
expect(res.status).toBe(404)
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/me returns user role in team', async () => {
|
||||
@@ -244,7 +228,23 @@ describe('team api', () => {
|
||||
permission: Permission.Read,
|
||||
}),
|
||||
)
|
||||
expect(mockGetSettings).toHaveBeenCalledWith('t1')
|
||||
expect(mockGetSettings).toHaveBeenCalledWith('t1', { isAdmin: true })
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/settings passes isAdmin: false when user lacks admin permission', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockImplementation(async (params) => {
|
||||
if (params.permission === Permission.Admin) {
|
||||
throw new Error('Forbidden')
|
||||
}
|
||||
})
|
||||
mockGetSettings.mockResolvedValue({
|
||||
archiveStorage: { enabled: true },
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/settings')
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockGetSettings).toHaveBeenCalledWith('t1', { isAdmin: false })
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/settings does not return emailNotification to readers', async () => {
|
||||
@@ -381,6 +381,65 @@ describe('team api', () => {
|
||||
expect(mockUpdateSettings).toHaveBeenCalledWith('t1', 'appearance.hideAgent', true)
|
||||
})
|
||||
|
||||
it('PATCH /teams/:teamId/settings updates autoArchive settings object', async () => {
|
||||
mockUpdateSettings.mockResolvedValue({
|
||||
autoArchive: {
|
||||
enabled: true,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/settings', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
key: 'autoArchive',
|
||||
value: {
|
||||
enabled: true,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.autoArchive.enabled).toBe(true)
|
||||
expect(data.autoArchive.days).toBe(30)
|
||||
expect(mockUpdateSettings).toHaveBeenCalledWith('t1', 'autoArchive', {
|
||||
enabled: true,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
})
|
||||
})
|
||||
|
||||
it('PATCH /teams/:teamId/settings updates autoArchive.enabled individual field', async () => {
|
||||
mockUpdateSettings.mockResolvedValue({
|
||||
autoArchive: {
|
||||
enabled: false,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/settings', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
key: 'autoArchive.enabled',
|
||||
value: false,
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockUpdateSettings).toHaveBeenCalledWith('t1', 'autoArchive.enabled', false)
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/user-metadata returns all metadata for the user in the team', async () => {
|
||||
mockListUserMetadata.mockResolvedValue([
|
||||
{ key: 'key1', value: 'value1' },
|
||||
|
||||
+11
-27
@@ -3,7 +3,6 @@ import { zValidator } from '@hono/zod-validator'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { teamService } from '@shumai/core/src/team/team'
|
||||
import { userMetadataService } from '@shumai/core/src/user-metadata/user-metadata'
|
||||
import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import {
|
||||
createTeamRequestSchema,
|
||||
getUserTeamsRequestSchema,
|
||||
@@ -31,7 +30,6 @@ import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { workflowTaskService } from '@shumai/core/src/workflowTask/workflowTask'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
import { NotificationType } from '@shumai/db'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
@@ -67,30 +65,6 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
},
|
||||
})
|
||||
})
|
||||
.post('/teams/:teamId/members', async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
|
||||
await teamService.joinTeam({
|
||||
teamId,
|
||||
userId: user.id,
|
||||
})
|
||||
|
||||
await notificationService.create({
|
||||
type: NotificationType.new_user_join_team,
|
||||
teamId,
|
||||
userId: user.id,
|
||||
})
|
||||
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.team_member_add,
|
||||
teamId,
|
||||
userId: user.id,
|
||||
itemId: user.id,
|
||||
})
|
||||
|
||||
return new Response(null, { status: 204 })
|
||||
})
|
||||
.get('/teams/:teamId/me', async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
@@ -210,7 +184,17 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const settings = await teamService.getSettings(teamId)
|
||||
const isAdmin = await authzService
|
||||
.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
.then(() => true)
|
||||
.catch(() => false)
|
||||
|
||||
const settings = await teamService.getSettings(teamId, { isAdmin })
|
||||
return c.json(settings)
|
||||
})
|
||||
.patch(
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { describe, expect, it, vi, beforeEach } from 'vitest'
|
||||
import { Hono } from 'hono'
|
||||
import uploadRoute from './upload'
|
||||
import uploadRoute, { localUploadRoute } from './upload'
|
||||
import { authMiddleware } from './middleware/auth'
|
||||
import { uploadService } from '@shumai/core/src/upload/upload'
|
||||
import * as s3Module from '@shumai/core/src/s3/s3'
|
||||
import type { CreateUploadTaskResponse, TaskInfo } from '@shumai/dtos'
|
||||
import type { PaginatedData } from '@shumai/core/src/pagination'
|
||||
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
|
||||
@@ -220,4 +221,38 @@ describe('Upload API', () => {
|
||||
key: 'files/test.mp4',
|
||||
})
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/upload/tasks/:taskId/cancel', async () => {
|
||||
vi.spyOn(uploadService, 'cancelUploadTask').mockResolvedValue({
|
||||
success: true,
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/team1/upload/tasks/task1/cancel', {
|
||||
method: 'POST',
|
||||
headers: { Authorization: 'Bearer test' },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json).toEqual({ success: true })
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: ResourceType.Team,
|
||||
id: 'team1',
|
||||
permission: Permission.Edit,
|
||||
}),
|
||||
)
|
||||
expect(uploadService.cancelUploadTask).toHaveBeenCalledWith('team1', 'user1', 'task1')
|
||||
})
|
||||
|
||||
describe('PUT /upload/local', () => {
|
||||
it('returns 404 when STORAGE_BACKEND is s3', async () => {
|
||||
vi.spyOn(s3Module, 'getStorageBackend').mockReturnValue('s3')
|
||||
const localApp = new Hono().route('/', localUploadRoute)
|
||||
const res = await localApp.request('/upload/local?bucket=b&key=k&Signature=sig', {
|
||||
method: 'PUT',
|
||||
})
|
||||
expect(res.status).toBe(404)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -14,7 +14,7 @@ import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { NotificationType } from '@shumai/db'
|
||||
import { s3Service, verifyLocalUrlSignature } from '@shumai/core/src/s3/s3'
|
||||
import { getStorageBackend, s3Service, verifyLocalUrlSignature } from '@shumai/core/src/s3/s3'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
@@ -23,6 +23,10 @@ export const localUploadRoute = new Hono().put(
|
||||
'/upload/local',
|
||||
zValidator('query', localUploadQuerySchema),
|
||||
async (c) => {
|
||||
if (getStorageBackend() !== 'local') {
|
||||
return c.text('Not found', 404)
|
||||
}
|
||||
|
||||
const { bucket, key, Signature } = c.req.valid('query')
|
||||
|
||||
if (!verifyLocalUrlSignature(bucket, key, Signature)) {
|
||||
@@ -163,5 +167,20 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
return c.json(resp)
|
||||
},
|
||||
)
|
||||
.post('/teams/:teamId/upload/tasks/:taskId/cancel', async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const taskId = c.req.param('taskId')
|
||||
const user = c.get('user')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const resp = await uploadService.cancelUploadTask(teamId, user.id, taskId)
|
||||
return c.json(resp)
|
||||
})
|
||||
|
||||
export default route
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/core",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -217,6 +217,91 @@ describe('download_asset Operation', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects downloading original of an archived asset with 409, while allowing proxy: true when a proxy exists', async () => {
|
||||
const clip = await createFile('archived.mov', {
|
||||
key: 'files/up-arch/archived.mov',
|
||||
media: {
|
||||
duration: 5,
|
||||
filesize: 512,
|
||||
frames: 150,
|
||||
finishedAt: '2026-01-01T00:00:00Z',
|
||||
metadata: null,
|
||||
original: null,
|
||||
imageTranscodes: [],
|
||||
proxyType: 'video',
|
||||
videoTranscodes: [
|
||||
{
|
||||
resolution: '720p',
|
||||
key: 'files/up-arch/archived-720p.mp4',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: clip.id },
|
||||
data: { status: 'archived' },
|
||||
})
|
||||
|
||||
// proxy: false is blocked with 409
|
||||
await expect(
|
||||
runOperation(downloadAssetOp, as(owner), { assetIds: [clip.id], proxy: false }),
|
||||
).rejects.toMatchObject({ status: 409 })
|
||||
|
||||
// proxy: true succeeds using the MP4 proxy in main storage
|
||||
const { files } = await runOperation(downloadAssetOp, as(owner), {
|
||||
assetIds: [clip.id],
|
||||
proxy: true,
|
||||
})
|
||||
expect(files).toHaveLength(1)
|
||||
expect(files[0]).toMatchObject({
|
||||
assetId: clip.id,
|
||||
kind: 'proxy',
|
||||
name: 'archived.mp4',
|
||||
})
|
||||
})
|
||||
|
||||
it('downloads PDF rendition when proxy: true is requested for Office documents with a pdfTranscode', async () => {
|
||||
const doc = await createFile('presentation.pptx', {
|
||||
key: 'files/up-office/presentation.pptx',
|
||||
media: {
|
||||
duration: 0,
|
||||
filesize: 1024,
|
||||
frames: 0,
|
||||
finishedAt: '2026-01-01T00:00:00Z',
|
||||
metadata: null,
|
||||
original: null,
|
||||
imageTranscodes: [],
|
||||
videoTranscodes: [],
|
||||
proxyType: 'pdf',
|
||||
pdfTranscode: {
|
||||
key: 'files/up-office/presentation.pdf',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const originalDownload = await runOperation(downloadAssetOp, as(owner), {
|
||||
assetIds: [doc.id],
|
||||
proxy: false,
|
||||
})
|
||||
expect(originalDownload.files[0]).toMatchObject({
|
||||
assetId: doc.id,
|
||||
kind: 'original',
|
||||
name: 'presentation.pptx',
|
||||
})
|
||||
|
||||
const proxyDownload = await runOperation(downloadAssetOp, as(owner), {
|
||||
assetIds: [doc.id],
|
||||
proxy: true,
|
||||
})
|
||||
expect(proxyDownload.files[0]).toMatchObject({
|
||||
assetId: doc.id,
|
||||
kind: 'proxy',
|
||||
name: 'presentation.pdf',
|
||||
})
|
||||
})
|
||||
|
||||
it.each([[[]], [Array.from({ length: 51 }, (_, i) => `a-${i}`)]])(
|
||||
'rejects %# asset ID lists outside 1 to 50',
|
||||
async (assetIds) => {
|
||||
|
||||
@@ -30,6 +30,9 @@ export function proxyKeyOf(media: PrismaJson.MediaInfo | null): string | undefin
|
||||
if (media.proxyType === 'video' || media.videoTranscodes?.length || media.videoPreview?.key) {
|
||||
return media.videoTranscodes?.[0]?.key || media.videoPreview?.key || undefined
|
||||
}
|
||||
if (media.pdfTranscode?.key && media.pdfTranscode.key !== media.original?.key) {
|
||||
return media.pdfTranscode.key
|
||||
}
|
||||
return media.original?.key || undefined
|
||||
}
|
||||
|
||||
@@ -66,6 +69,19 @@ export class AssetDownloadService {
|
||||
})
|
||||
}
|
||||
const isOriginal = key === originalKey
|
||||
const isArchived =
|
||||
asset.status === 'archiving' ||
|
||||
asset.status === 'archived' ||
|
||||
asset.status === 'restoring' ||
|
||||
Boolean(asset.storageKey?.isArchived)
|
||||
|
||||
if (isArchived && (!opts.proxy || isOriginal)) {
|
||||
throw new HTTPException(409, {
|
||||
message: opts.proxy
|
||||
? `Asset ${assetId} ("${asset.name}") is archived and has no proxy file to download. Restore the asset first.`
|
||||
: `Original file for asset ${assetId} ("${asset.name}") is archived. Restore the asset or request proxy: true to download its proxy.`,
|
||||
})
|
||||
}
|
||||
const name = nameForKey(asset.name || 'file', key, isOriginal)
|
||||
|
||||
const bucket = storageBucket()
|
||||
|
||||
@@ -244,6 +244,60 @@ describe('AssetReadService', () => {
|
||||
expect(s3Service.getObject).not.toHaveBeenCalled()
|
||||
expect(outcome.status === 'ready' && outcome.result.text?.content).toBe('')
|
||||
})
|
||||
|
||||
it('rejects reading raw text or native PDF pages when the original is archived, while allowing proxy.pdf', async () => {
|
||||
const archivedText = await createFile('notes.md', {
|
||||
mediaType: 'text/markdown',
|
||||
key: 'raw/notes.md',
|
||||
status: 'archived',
|
||||
})
|
||||
await expect(
|
||||
assetReadService.read(
|
||||
user(owner),
|
||||
{ assetId: archivedText.id, docConfig: { mode: 'text' } },
|
||||
now,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
status: 400,
|
||||
message: expect.stringContaining('is archived in cold storage'),
|
||||
})
|
||||
|
||||
const archivedNativePdf = await createFile('report.pdf', {
|
||||
mediaType: 'application/pdf',
|
||||
key: 'raw/report.pdf',
|
||||
status: 'archived',
|
||||
media: {
|
||||
proxyType: 'pdf',
|
||||
pdfTranscode: { key: 'raw/report.pdf', numPages: 3 },
|
||||
} as unknown as PrismaJson.MediaInfo,
|
||||
})
|
||||
await expect(
|
||||
assetReadService.read(
|
||||
user(owner),
|
||||
{ assetId: archivedNativePdf.id, docConfig: { mode: 'pages', startPage: 1, endPage: 1 } },
|
||||
now,
|
||||
),
|
||||
).rejects.toMatchObject({
|
||||
status: 400,
|
||||
message: expect.stringContaining('is archived in cold storage'),
|
||||
})
|
||||
|
||||
const archivedOfficeDoc = await createFile('deck.pptx', {
|
||||
mediaType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
key: 'raw/deck.pptx',
|
||||
status: 'archived',
|
||||
media: {
|
||||
proxyType: 'pdf',
|
||||
pdfTranscode: { key: 'transcoded/deck/proxy.pdf', numPages: 3 },
|
||||
} as unknown as PrismaJson.MediaInfo,
|
||||
})
|
||||
const officeOutcome = await assetReadService.read(
|
||||
user(owner),
|
||||
{ assetId: archivedOfficeDoc.id, docConfig: { mode: 'pages', startPage: 1, endPage: 1 } },
|
||||
now,
|
||||
)
|
||||
expect(officeOutcome.status).toBe('pending')
|
||||
})
|
||||
})
|
||||
|
||||
describe('render tasks', () => {
|
||||
|
||||
@@ -55,6 +55,11 @@ function toReadable(asset: AssetRecord): ReadableAsset {
|
||||
filenameLower: asset.name?.toLowerCase() || '',
|
||||
storageKey: asset.storageKey?.key ?? null,
|
||||
media: asset.media as PrismaJson.MediaInfo | null,
|
||||
isArchived:
|
||||
asset.status === 'archiving' ||
|
||||
asset.status === 'archived' ||
|
||||
asset.status === 'restoring' ||
|
||||
Boolean(asset.storageKey?.isArchived),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -107,9 +107,23 @@ export const documentReader: AssetReader<'document'> = {
|
||||
`Asset "${asset.name}" is a binary PDF/document and cannot be read as raw text. Please call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ... } to view its visual pages.`,
|
||||
)
|
||||
}
|
||||
if (asset.isArchived) {
|
||||
throw badRequest(
|
||||
`Asset "${asset.name}" is archived in cold storage and cannot be read as raw text. Restore the asset first, or use docConfig: { mode: "pages", startPage: 1, endPage: ... } if it has a generated PDF proxy.`,
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (
|
||||
asset.isArchived &&
|
||||
(!asset.media?.pdfTranscode?.key || asset.media.pdfTranscode.key === asset.storageKey)
|
||||
) {
|
||||
throw badRequest(
|
||||
`Asset "${asset.name}" is a native PDF whose original file is archived in cold storage. Restore the asset first to render its pages.`,
|
||||
)
|
||||
}
|
||||
|
||||
if (startPage === null || endPage === null) {
|
||||
throw badRequest('startPage and endPage are required when docConfig mode is "pages".')
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { WorkflowTaskType } from '@shumai/db'
|
||||
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
|
||||
import type { AssetReader } from '../types'
|
||||
import { badRequest, type AssetReader } from '../types'
|
||||
|
||||
/** Images (including PSD): the web proxy, or a copy with the annotation drawn on top. */
|
||||
export const imageReader: AssetReader<'image'> = {
|
||||
@@ -16,7 +16,13 @@ export const imageReader: AssetReader<'image'> = {
|
||||
validate() {},
|
||||
|
||||
async plan(asset, _options, { annotation }) {
|
||||
const key = asset.media?.imageTranscodes?.[0]?.key || asset.storageKey
|
||||
const proxyKey = asset.media?.imageTranscodes?.[0]?.key
|
||||
if (!proxyKey && asset.isArchived) {
|
||||
throw badRequest(
|
||||
`Asset "${asset.name}" is archived in cold storage and has no web image proxy. Restore the asset first.`,
|
||||
)
|
||||
}
|
||||
const key = proxyKey || asset.storageKey
|
||||
if (!key) {
|
||||
throw new Error('No media content found for this asset.')
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ export interface ReadableAsset {
|
||||
filenameLower: string
|
||||
storageKey: string | null
|
||||
media: PrismaJson.MediaInfo | null
|
||||
/** True when the original raw file is archived, archiving, or restoring. */
|
||||
isArchived?: boolean
|
||||
}
|
||||
|
||||
export interface ReadAnnotation {
|
||||
|
||||
@@ -0,0 +1,517 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import {
|
||||
AssetStatus,
|
||||
AssetType,
|
||||
prisma,
|
||||
WorkflowTaskStatus,
|
||||
WorkflowTaskType,
|
||||
type User,
|
||||
} from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
import { assetService } from './asset'
|
||||
import { searchService } from '../search/search'
|
||||
import { resetArchiveStorageServiceForTests, getArchiveStorageService } from '../s3/archive-storage'
|
||||
|
||||
describe('AssetService Archive & Restore', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
const archiveRoot = path.join(process.cwd(), 'data-test-asset-archive')
|
||||
let originalEnv: NodeJS.ProcessEnv
|
||||
let owner: User
|
||||
let teamId: string
|
||||
let projectId: string
|
||||
let rootFolderId: string
|
||||
|
||||
beforeEach(async () => {
|
||||
originalEnv = { ...process.env }
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
|
||||
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
|
||||
resetArchiveStorageServiceForTests()
|
||||
|
||||
if (fs.existsSync(archiveRoot)) {
|
||||
fs.rmSync(archiveRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
owner = await prisma.user.create({
|
||||
data: { name: 'Archive Owner', email: 'archive-owner@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 'Archive Team' } })
|
||||
teamId = team.id
|
||||
await prisma.teamMember.create({ data: { teamId, userId: owner.id, role: 'owner' } })
|
||||
|
||||
const project = await projectService.createProject(owner, { teamId, name: 'Archive Project' })
|
||||
projectId = project.id
|
||||
rootFolderId = project.rootFolder!
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
resetArchiveStorageServiceForTests()
|
||||
if (fs.existsSync(archiveRoot)) {
|
||||
fs.rmSync(archiveRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
async function createProcessedFile(params: {
|
||||
name: string
|
||||
parentId?: string
|
||||
key: string
|
||||
media?: Partial<PrismaJson.MediaInfo>
|
||||
isArchived?: boolean
|
||||
status?: AssetStatus
|
||||
}) {
|
||||
const asset = await assetService.createAsset({
|
||||
name: params.name,
|
||||
type: 'file',
|
||||
parentId: params.parentId ?? rootFolderId,
|
||||
projectId,
|
||||
key: params.key,
|
||||
sizeByte: 1024,
|
||||
contentType: 'video/mp4',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
|
||||
if (params.isArchived) {
|
||||
await prisma.storageKey.update({
|
||||
where: { key: params.key },
|
||||
data: { isArchived: true },
|
||||
})
|
||||
}
|
||||
|
||||
const fullMedia: PrismaJson.MediaInfo | undefined = params.media
|
||||
? {
|
||||
duration: 0,
|
||||
filesize: 1024,
|
||||
frames: 0,
|
||||
finishedAt: '2026-01-01T00:00:00Z',
|
||||
metadata: null,
|
||||
original: null,
|
||||
imageTranscodes: [],
|
||||
videoTranscodes: [],
|
||||
...params.media,
|
||||
}
|
||||
: undefined
|
||||
|
||||
await prisma.asset.update({
|
||||
where: { id: asset.id },
|
||||
data: {
|
||||
status: params.status ?? (params.isArchived ? AssetStatus.archived : AssetStatus.processed),
|
||||
...(fullMedia ? { media: fullMedia } : {}),
|
||||
},
|
||||
})
|
||||
|
||||
return prisma.asset.findUniqueOrThrow({
|
||||
where: { id: asset.id },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
}
|
||||
|
||||
it('rejects archive and restore when ARCHIVE_STORAGE_BACKEND is not configured', async () => {
|
||||
delete process.env.ARCHIVE_STORAGE_BACKEND
|
||||
resetArchiveStorageServiceForTests()
|
||||
|
||||
const file = await createProcessedFile({
|
||||
name: 'clip.mp4',
|
||||
key: 'files/01TEST/clip.mp4',
|
||||
})
|
||||
|
||||
await expect(assetService.archiveAssets([file.id])).rejects.toMatchObject({
|
||||
status: 400,
|
||||
message: 'Archive storage is not configured',
|
||||
})
|
||||
await expect(assetService.restoreArchivedAssets([file.id])).rejects.toMatchObject({
|
||||
status: 400,
|
||||
message: 'Archive storage is not configured',
|
||||
})
|
||||
})
|
||||
|
||||
it('archives and restores standalone files, deduplicating shared StorageKey across copies and updating all copies', async () => {
|
||||
const file = await createProcessedFile({
|
||||
name: 'shared.mp4',
|
||||
key: 'files/01SHARED/shared.mp4',
|
||||
})
|
||||
|
||||
const subFolder = await assetService.createAsset({
|
||||
name: 'Copies',
|
||||
type: 'folder',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
creatorId: owner.id,
|
||||
})
|
||||
const [copiedId] = await assetService.copyAssets({
|
||||
assetIds: [file.id],
|
||||
newParentId: subFolder.id,
|
||||
creatorId: owner.id,
|
||||
withComments: false,
|
||||
})
|
||||
|
||||
// Archive original -> both original and copy share the StorageKey, so both transition to 'archiving' with 1 WorkflowTask
|
||||
const archiveRes = await assetService.archiveAssets([file.id, copiedId])
|
||||
expect(archiveRes.archivedAssetIds.sort()).toEqual([file.id, copiedId].sort())
|
||||
expect(archiveRes.taskIds).toHaveLength(1)
|
||||
expect(archiveRes.skippedCount).toBe(0)
|
||||
|
||||
const updatedRows = await prisma.asset.findMany({
|
||||
where: { id: { in: [file.id, copiedId] } },
|
||||
})
|
||||
expect(updatedRows.map((r) => r.status)).toEqual([AssetStatus.archiving, AssetStatus.archiving])
|
||||
|
||||
const task = await prisma.workflowTask.findUniqueOrThrow({
|
||||
where: { id: archiveRes.taskIds[0] },
|
||||
})
|
||||
expect(task.type).toBe(WorkflowTaskType.archive_asset)
|
||||
expect(task.payload?.archive?.storageKeyId).toBe(file.storageKeyId)
|
||||
|
||||
// Simulate workflow completion -> StorageKey.isArchived = true, Asset.status = 'archived'
|
||||
await prisma.storageKey.update({
|
||||
where: { id: file.storageKeyId! },
|
||||
data: { isArchived: true },
|
||||
})
|
||||
await prisma.asset.updateMany({
|
||||
where: { storageKeyId: file.storageKeyId! },
|
||||
data: { status: AssetStatus.archived },
|
||||
})
|
||||
await prisma.workflowTask.update({
|
||||
where: { id: task.id },
|
||||
data: { status: WorkflowTaskStatus.completed },
|
||||
})
|
||||
|
||||
// Restore from archive via the copy -> both transition to 'restoring' with 1 restore_asset task
|
||||
const restoreRes = await assetService.restoreArchivedAssets([copiedId])
|
||||
expect(restoreRes.restoredAssetIds.sort()).toEqual([file.id, copiedId].sort())
|
||||
expect(restoreRes.taskIds).toHaveLength(1)
|
||||
|
||||
const restoringRows = await prisma.asset.findMany({
|
||||
where: { id: { in: [file.id, copiedId] } },
|
||||
})
|
||||
expect(restoringRows.map((r) => r.status)).toEqual([
|
||||
AssetStatus.restoring,
|
||||
AssetStatus.restoring,
|
||||
])
|
||||
})
|
||||
|
||||
it('recursively archives and restores folders and version stacks without mutating folder or version_stack DB status', async () => {
|
||||
const folder = await assetService.createAsset({
|
||||
name: 'Shoot Day 1',
|
||||
type: 'folder',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
creatorId: owner.id,
|
||||
})
|
||||
|
||||
const v1 = await createProcessedFile({
|
||||
name: 'cut_v1.mp4',
|
||||
parentId: folder.id,
|
||||
key: 'files/01V1/cut_v1.mp4',
|
||||
})
|
||||
const v2 = await createProcessedFile({
|
||||
name: 'cut_v2.mp4',
|
||||
parentId: folder.id,
|
||||
key: 'files/01V2/cut_v2.mp4',
|
||||
})
|
||||
|
||||
// Stack v2 onto v1 to create a version_stack inside folder
|
||||
await assetService.reparentAssets({
|
||||
assetIds: [v2.id],
|
||||
newParentId: v1.id,
|
||||
creatorId: owner.id,
|
||||
})
|
||||
|
||||
const v1Reloaded = await prisma.asset.findUniqueOrThrow({ where: { id: v1.id } })
|
||||
const stackId = v1Reloaded.parentId!
|
||||
const stackRowBefore = await prisma.asset.findUniqueOrThrow({ where: { id: stackId } })
|
||||
expect(stackRowBefore.type).toBe(AssetType.version_stack)
|
||||
expect(stackRowBefore.status).toBe(AssetStatus.uploaded)
|
||||
|
||||
// Archive the parent folder
|
||||
const archiveRes = await assetService.archiveAssets([folder.id])
|
||||
expect(archiveRes.archivedAssetIds.sort()).toEqual([v1.id, v2.id].sort())
|
||||
expect(archiveRes.taskIds).toHaveLength(2)
|
||||
|
||||
// Verify folder and version_stack DB statuses remain 'uploaded'
|
||||
const folderAfter = await prisma.asset.findUniqueOrThrow({ where: { id: folder.id } })
|
||||
const stackAfter = await prisma.asset.findUniqueOrThrow({ where: { id: stackId } })
|
||||
expect(folderAfter.status).toBe(AssetStatus.uploaded)
|
||||
expect(stackAfter.status).toBe(AssetStatus.uploaded)
|
||||
|
||||
// Mark both versions as 'archived' and verify version_stack AssetInfo reflects latestVersion status ('archived')
|
||||
await prisma.storageKey.updateMany({
|
||||
where: { id: { in: [v1.storageKeyId!, v2.storageKeyId!] } },
|
||||
data: { isArchived: true },
|
||||
})
|
||||
await prisma.asset.updateMany({
|
||||
where: { id: { in: [v1.id, v2.id] } },
|
||||
data: { status: AssetStatus.archived },
|
||||
})
|
||||
|
||||
const stackInfo = await assetService.getAsset({ assetId: stackId })
|
||||
expect(stackInfo.status).toBe(AssetStatus.archived)
|
||||
expect(stackInfo.versionStack?.versions.map((v) => v.status)).toEqual([
|
||||
AssetStatus.archived,
|
||||
AssetStatus.archived,
|
||||
])
|
||||
})
|
||||
|
||||
it('omits original.url and native PDF pdfTranscode.url when archived, while keeping Office proxy.pdf url', async () => {
|
||||
const nativePdf = await createProcessedFile({
|
||||
name: 'script.pdf',
|
||||
key: 'files/01PDF/script.pdf',
|
||||
isArchived: true,
|
||||
media: {
|
||||
proxyType: 'pdf',
|
||||
original: { key: 'files/01PDF/script.pdf', filesizeInBytes: 1024, codec: '' },
|
||||
pdfTranscode: { key: 'files/01PDF/script.pdf' },
|
||||
videoTranscodes: [],
|
||||
imageTranscodes: [],
|
||||
},
|
||||
})
|
||||
|
||||
const officeDoc = await createProcessedFile({
|
||||
name: 'deck.pptx',
|
||||
key: 'files/01PPTX/deck.pptx',
|
||||
isArchived: true,
|
||||
media: {
|
||||
proxyType: 'pdf',
|
||||
original: { key: 'files/01PPTX/deck.pptx', filesizeInBytes: 2048, codec: '' },
|
||||
pdfTranscode: { key: 'files/01PPTX/proxy.pdf' },
|
||||
videoTranscodes: [],
|
||||
imageTranscodes: [],
|
||||
},
|
||||
})
|
||||
|
||||
const nativePdfInfo = await assetService.getAsset({ assetId: nativePdf.id })
|
||||
expect(nativePdfInfo.media?.original?.key).toBe('files/01PDF/script.pdf')
|
||||
expect(nativePdfInfo.media?.pdfTranscode?.url).toBeUndefined()
|
||||
|
||||
const officeDocInfo = await assetService.getAsset({ assetId: officeDoc.id })
|
||||
expect(officeDocInfo.media?.original?.key).toBe('files/01PPTX/deck.pptx')
|
||||
expect(officeDocInfo.media?.pdfTranscode?.url).toBeDefined()
|
||||
})
|
||||
|
||||
it('blocks original file download via getDownloadUrl with 409 when archived, while allowing proxy keys, and skips archived files in getDownloadLinks', async () => {
|
||||
const archivedVideo = await createProcessedFile({
|
||||
name: 'archived.mp4',
|
||||
key: 'files/01VID/archived.mp4',
|
||||
isArchived: true,
|
||||
media: {
|
||||
proxyType: 'video',
|
||||
original: { key: 'files/01VID/archived.mp4', filesizeInBytes: 1024, codec: 'h264' },
|
||||
videoTranscodes: [
|
||||
{
|
||||
resolution: '720p',
|
||||
key: 'files/01VID/archived-720p.mp4',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
},
|
||||
],
|
||||
imageTranscodes: [],
|
||||
},
|
||||
})
|
||||
const hotVideo = await createProcessedFile({
|
||||
name: 'hot.mp4',
|
||||
key: 'files/01HOT/hot.mp4',
|
||||
isArchived: false,
|
||||
})
|
||||
|
||||
// Original key is blocked with 409
|
||||
await expect(
|
||||
assetService.getDownloadUrl(archivedVideo.id, 'files/01VID/archived.mp4'),
|
||||
).rejects.toMatchObject({
|
||||
status: 409,
|
||||
})
|
||||
|
||||
// Proxy key in same asset folder is allowed
|
||||
const proxyUrl = await assetService.getDownloadUrl(
|
||||
archivedVideo.id,
|
||||
'files/01VID/archived-720p.mp4',
|
||||
)
|
||||
expect(proxyUrl).toContain('archived-720p.mp4')
|
||||
|
||||
// Batch getDownloadLinks skips archivedVideo and returns hotVideo + skippedArchivedCount = 1
|
||||
const links = await assetService.getDownloadLinks([archivedVideo.id, hotVideo.id])
|
||||
expect(links.files).toHaveLength(1)
|
||||
expect(links.files[0].id).toBe(hotVideo.id)
|
||||
expect(links.skippedArchivedCount).toBe(1)
|
||||
})
|
||||
|
||||
it('preserves archived status when restoring from Trash and when copying assets, and purges archive storage during GC', async () => {
|
||||
const archivedFile = await createProcessedFile({
|
||||
name: 'cold.mp4',
|
||||
key: 'files/01COLD/cold.mp4',
|
||||
isArchived: true,
|
||||
})
|
||||
|
||||
// Write physical file into local archive storage
|
||||
const archiveStorage = getArchiveStorageService()
|
||||
await archiveStorage.uploadStream(
|
||||
'files/01COLD/cold.mp4',
|
||||
new ReadableStream({
|
||||
start(c) {
|
||||
c.enqueue(new TextEncoder().encode('cold bytes'))
|
||||
c.close()
|
||||
},
|
||||
}),
|
||||
10,
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
// Copy archived asset -> new copy has status 'archived'
|
||||
const targetFolder = await assetService.createAsset({
|
||||
name: 'Target',
|
||||
type: 'folder',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
creatorId: owner.id,
|
||||
})
|
||||
const [copiedId] = await assetService.copyAssets({
|
||||
assetIds: [archivedFile.id],
|
||||
newParentId: targetFolder.id,
|
||||
creatorId: owner.id,
|
||||
withComments: false,
|
||||
})
|
||||
const copiedRow = await prisma.asset.findUniqueOrThrow({ where: { id: copiedId } })
|
||||
expect(copiedRow.status).toBe(AssetStatus.archived)
|
||||
|
||||
// Delete and restore from Trash -> status returns to 'archived' (not 'processed')
|
||||
await assetService.deleteAssets([archivedFile.id])
|
||||
const trashedRow = await prisma.asset.findUniqueOrThrow({ where: { id: archivedFile.id } })
|
||||
expect(trashedRow.status).toBe(AssetStatus.trashed)
|
||||
|
||||
await assetService.restoreAssets([archivedFile.id])
|
||||
const restoredFromTrash = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: archivedFile.id },
|
||||
})
|
||||
expect(restoredFromTrash.status).toBe(AssetStatus.archived)
|
||||
|
||||
// Filter via SearchService with isArchived condition
|
||||
const archivedSearch = await searchService.search(rootFolderId, {
|
||||
operator: 'AND',
|
||||
conditions: [{ field: 'isArchived', operator: 'eq', value: true }],
|
||||
recursively: true,
|
||||
isSemantic: false,
|
||||
})
|
||||
expect(archivedSearch.data.map((d) => d.id).sort()).toEqual([archivedFile.id, copiedId].sort())
|
||||
|
||||
const unarchivedSearch = await searchService.search(rootFolderId, {
|
||||
operator: 'AND',
|
||||
conditions: [{ field: 'isArchived', operator: 'eq', value: false }],
|
||||
recursively: true,
|
||||
isSemantic: false,
|
||||
})
|
||||
expect(unarchivedSearch.data.map((d) => d.id)).toEqual([])
|
||||
|
||||
// Purge both assets and run GC -> file is deleted from archive storage too
|
||||
await assetService.deleteAssets([archivedFile.id, copiedId])
|
||||
await prisma.storageKey.update({
|
||||
where: { id: archivedFile.storageKeyId! },
|
||||
data: { createdAt: new Date(Date.now() - 60_000) },
|
||||
})
|
||||
await assetService.emptyTrash(projectId)
|
||||
|
||||
await expect(archiveStorage.headObject('files/01COLD/cold.mp4')).rejects.toThrow(/NoSuchKey/)
|
||||
const skAfter = await prisma.storageKey.findUnique({
|
||||
where: { id: archivedFile.storageKeyId! },
|
||||
})
|
||||
expect(skAfter).toBeNull()
|
||||
})
|
||||
|
||||
it('refuses to archive when a sibling copy sharing storageKeyId is processing or has an active task', async () => {
|
||||
const originalFile = await createProcessedFile({
|
||||
name: 'original.mp4',
|
||||
key: 'files/01SIB/original.mp4',
|
||||
})
|
||||
|
||||
// Sibling copy sharing the same storageKeyId
|
||||
const siblingFile = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'sibling.mp4',
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.processing,
|
||||
storageKeyId: originalFile.storageKeyId,
|
||||
projectId,
|
||||
parentId: rootFolderId,
|
||||
creatorId: owner.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Trying to archive originalFile when siblingFile is processing throws 409
|
||||
await expect(assetService.archiveAssets([originalFile.id])).rejects.toMatchObject({
|
||||
status: 409,
|
||||
message: 'Only processed assets can be archived',
|
||||
})
|
||||
|
||||
// Now set sibling to processed, but give it an active workflowTask
|
||||
await prisma.asset.update({
|
||||
where: { id: siblingFile.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
})
|
||||
const activeTask = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: siblingFile.id,
|
||||
type: WorkflowTaskType.transcode,
|
||||
status: WorkflowTaskStatus.processing,
|
||||
projectId,
|
||||
teamId,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(assetService.archiveAssets([originalFile.id])).rejects.toMatchObject({
|
||||
status: 409,
|
||||
message: 'Cannot archive asset while a workflow task is in progress',
|
||||
})
|
||||
|
||||
// Once task completes, archive succeeds
|
||||
await prisma.workflowTask.update({
|
||||
where: { id: activeTask.id },
|
||||
data: { status: WorkflowTaskStatus.completed },
|
||||
})
|
||||
const res = await assetService.archiveAssets([originalFile.id])
|
||||
expect(res.archivedAssetIds.sort()).toEqual([originalFile.id, siblingFile.id].sort())
|
||||
})
|
||||
|
||||
it('ignores symlinks inside folders during recursive folder archiving', async () => {
|
||||
const folder = await assetService.createAsset({
|
||||
name: 'archive-folder',
|
||||
type: 'folder',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
creatorId: owner.id,
|
||||
})
|
||||
|
||||
const realFile = await createProcessedFile({
|
||||
name: 'nested-real.mp4',
|
||||
key: 'files/01NEST/nested-real.mp4',
|
||||
parentId: folder.id,
|
||||
})
|
||||
|
||||
// External target file
|
||||
const externalFile = await createProcessedFile({
|
||||
name: 'external.mp4',
|
||||
key: 'files/01EXT/external.mp4',
|
||||
parentId: rootFolderId,
|
||||
})
|
||||
|
||||
// Create a symlink inside the folder pointing to externalFile
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'symlink-inside.mp4',
|
||||
type: AssetType.symlink,
|
||||
status: AssetStatus.processed,
|
||||
projectId,
|
||||
parentId: folder.id,
|
||||
targetId: externalFile.id,
|
||||
creatorId: owner.id,
|
||||
},
|
||||
})
|
||||
|
||||
const res = await assetService.archiveAssets([folder.id])
|
||||
// Only realFile should be archived, NOT externalFile
|
||||
expect(res.archivedAssetIds).toEqual([realFile.id])
|
||||
|
||||
const extRow = await prisma.asset.findUniqueOrThrow({ where: { id: externalFile.id } })
|
||||
expect(extRow.status).toBe(AssetStatus.processed)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,520 @@
|
||||
import {
|
||||
AssetStatus,
|
||||
AssetType,
|
||||
Prisma,
|
||||
prisma,
|
||||
type PrismaClient,
|
||||
WorkflowTaskStatus,
|
||||
WorkflowTaskType,
|
||||
} from '@shumai/db'
|
||||
import type { ArchiveAssetsResponse, RestoreArchivedAssetsResponse } from '@shumai/dtos'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { isArchiveStorageEnabled } from '@shumai/core/src/s3/archive-storage'
|
||||
|
||||
export class AssetArchiveService {
|
||||
constructor(private readonly prismaClient: PrismaClient = prisma) {}
|
||||
|
||||
async resolveCandidateFileAssetsForArchiveOrRestore(assetIds: string[]) {
|
||||
if (assetIds.length === 0) return []
|
||||
|
||||
const topAssets = await this.prismaClient.asset.findMany({
|
||||
where: { id: { in: assetIds }, isDeleted: false },
|
||||
select: { id: true, type: true },
|
||||
})
|
||||
|
||||
const foundIds = new Set(topAssets.map((a) => a.id))
|
||||
const uniqueRequested = Array.from(new Set(assetIds))
|
||||
if (foundIds.size !== uniqueRequested.length) {
|
||||
throw new HTTPException(404, { message: 'Asset not found' })
|
||||
}
|
||||
|
||||
if (topAssets.some((a) => a.type === AssetType.symlink)) {
|
||||
throw new HTTPException(400, { message: 'Symlinks cannot be archived or restored' })
|
||||
}
|
||||
|
||||
const startingIds = topAssets.map((a) => a.id)
|
||||
|
||||
const expandIdsRecursively = async (rootIds: string[]) => {
|
||||
if (rootIds.length === 0) return []
|
||||
return this.prismaClient.$queryRaw<
|
||||
Array<{
|
||||
id: string
|
||||
type: string
|
||||
}>
|
||||
>`
|
||||
WITH RECURSIVE descendant AS (
|
||||
SELECT id, type::text AS type, is_deleted
|
||||
FROM assets
|
||||
WHERE id IN (${Prisma.join(rootIds)}) AND is_deleted = false AND type != 'symlink'
|
||||
UNION ALL
|
||||
SELECT a.id, a.type::text AS type, a.is_deleted
|
||||
FROM assets a
|
||||
INNER JOIN descendant d ON a.parent_id = d.id
|
||||
WHERE a.is_deleted = false AND a.type != 'symlink'
|
||||
)
|
||||
SELECT id, type FROM descendant;
|
||||
`
|
||||
}
|
||||
|
||||
const descendants = await expandIdsRecursively(startingIds)
|
||||
const fileIds = Array.from(
|
||||
new Set(descendants.filter((r) => r.type === AssetType.file).map((r) => r.id)),
|
||||
)
|
||||
|
||||
if (fileIds.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
return this.prismaClient.asset.findMany({
|
||||
where: {
|
||||
id: { in: fileIds },
|
||||
isDeleted: false,
|
||||
type: AssetType.file,
|
||||
status: {
|
||||
notIn: [AssetStatus.trashed, AssetStatus.pending_purge],
|
||||
},
|
||||
},
|
||||
include: {
|
||||
storageKey: true,
|
||||
project: { select: { id: true, teamId: true } },
|
||||
},
|
||||
orderBy: { id: 'asc' },
|
||||
})
|
||||
}
|
||||
|
||||
async archiveAssets(
|
||||
assetIds: string[],
|
||||
options?: {
|
||||
suppressBusyErrors?: boolean
|
||||
autoArchivePolicy?: {
|
||||
method: 'not_updated' | 'after_created'
|
||||
cutoffDate: Date
|
||||
restoreCutoffDate: Date
|
||||
}
|
||||
},
|
||||
): Promise<ArchiveAssetsResponse> {
|
||||
if (!isArchiveStorageEnabled()) {
|
||||
throw new HTTPException(400, { message: 'Archive storage is not configured' })
|
||||
}
|
||||
if (assetIds.length === 0) {
|
||||
return { archivedAssetIds: [], taskIds: [], skippedCount: 0 }
|
||||
}
|
||||
|
||||
const candidateFiles = await this.resolveCandidateFileAssetsForArchiveOrRestore(assetIds)
|
||||
if (candidateFiles.length === 0) {
|
||||
return { archivedAssetIds: [], taskIds: [], skippedCount: 0 }
|
||||
}
|
||||
|
||||
const candidateStorageKeyIds = Array.from(
|
||||
new Set(candidateFiles.map((f) => f.storageKeyId).filter((id): id is string => !!id)),
|
||||
)
|
||||
|
||||
// Sibling copies sharing the same storageKeyId must also be protected
|
||||
const allFilesForKeys = await this.prismaClient.asset.findMany({
|
||||
where: {
|
||||
storageKeyId: { in: candidateStorageKeyIds },
|
||||
isDeleted: false,
|
||||
type: AssetType.file,
|
||||
status: { notIn: [AssetStatus.trashed, AssetStatus.pending_purge] },
|
||||
},
|
||||
select: { id: true, storageKeyId: true, status: true },
|
||||
})
|
||||
|
||||
const allFileIds = allFilesForKeys.map((f) => f.id)
|
||||
|
||||
// Check active transcode/watermark tasks across candidate files AND all siblings sharing storage keys
|
||||
const activeTasks = await this.prismaClient.workflowTask.findMany({
|
||||
where: {
|
||||
assetId: { in: allFileIds },
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
type: {
|
||||
in: [
|
||||
WorkflowTaskType.transcode,
|
||||
WorkflowTaskType.transcode_video,
|
||||
WorkflowTaskType.transcode_image,
|
||||
WorkflowTaskType.transcode_pdf,
|
||||
WorkflowTaskType.transcode_pdf_pages,
|
||||
WorkflowTaskType.transcode_screenshot,
|
||||
WorkflowTaskType.transcode_image_annotation,
|
||||
WorkflowTaskType.transcode_watermark,
|
||||
],
|
||||
},
|
||||
},
|
||||
select: { assetId: true },
|
||||
})
|
||||
const busyAssetIds = new Set(activeTasks.map((t) => t.assetId))
|
||||
|
||||
const storageKeysWithBusyTask = new Set<string>()
|
||||
const storageKeysWithUnprocessedSiblings = new Set<string>()
|
||||
|
||||
for (const f of allFilesForKeys) {
|
||||
if (!f.storageKeyId) continue
|
||||
if (busyAssetIds.has(f.id)) {
|
||||
storageKeysWithBusyTask.add(f.storageKeyId)
|
||||
}
|
||||
if (f.status === AssetStatus.uploading || f.status === AssetStatus.processing) {
|
||||
storageKeysWithUnprocessedSiblings.add(f.storageKeyId)
|
||||
}
|
||||
}
|
||||
|
||||
const eligibleByStorageKey = new Map<string, typeof candidateFiles>()
|
||||
let skippedCount = 0
|
||||
let alreadyArchivedCount = 0
|
||||
let invalidStateCount = 0
|
||||
let busyCount = 0
|
||||
|
||||
for (const file of candidateFiles) {
|
||||
if (!file.storageKeyId || !file.storageKey || !file.projectId || !file.project?.teamId) {
|
||||
skippedCount++
|
||||
continue
|
||||
}
|
||||
if (
|
||||
file.status === AssetStatus.archiving ||
|
||||
file.status === AssetStatus.archived ||
|
||||
file.storageKey.isArchived
|
||||
) {
|
||||
skippedCount++
|
||||
alreadyArchivedCount++
|
||||
continue
|
||||
}
|
||||
if (busyAssetIds.has(file.id) || storageKeysWithBusyTask.has(file.storageKeyId)) {
|
||||
skippedCount++
|
||||
busyCount++
|
||||
continue
|
||||
}
|
||||
if (
|
||||
file.status !== AssetStatus.processed ||
|
||||
storageKeysWithUnprocessedSiblings.has(file.storageKeyId)
|
||||
) {
|
||||
skippedCount++
|
||||
invalidStateCount++
|
||||
continue
|
||||
}
|
||||
|
||||
const list = eligibleByStorageKey.get(file.storageKeyId) || []
|
||||
list.push(file)
|
||||
eligibleByStorageKey.set(file.storageKeyId, list)
|
||||
}
|
||||
|
||||
if (eligibleByStorageKey.size === 0) {
|
||||
if (!options?.suppressBusyErrors) {
|
||||
if (alreadyArchivedCount === 0 && busyCount > 0) {
|
||||
throw new HTTPException(409, {
|
||||
message: 'Cannot archive asset while a workflow task is in progress',
|
||||
})
|
||||
}
|
||||
if (alreadyArchivedCount === 0 && invalidStateCount > 0) {
|
||||
throw new HTTPException(409, {
|
||||
message: 'Only processed assets can be archived',
|
||||
})
|
||||
}
|
||||
}
|
||||
return { archivedAssetIds: [], taskIds: [], skippedCount }
|
||||
}
|
||||
|
||||
const archivedAssetIdsSet = new Set<string>()
|
||||
const taskIds: string[] = []
|
||||
|
||||
for (const [storageKeyId, filesForKey] of eligibleByStorageKey.entries()) {
|
||||
const primaryFile = filesForKey[0]
|
||||
await this.prismaClient.$transaction(async (tx) => {
|
||||
// 1. Lock the storageKey row to serialize concurrent archive requests
|
||||
const lockedKeys = await tx.$queryRaw<Array<{ id: string; isArchived: boolean }>>`
|
||||
SELECT id, is_archived AS "isArchived"
|
||||
FROM storage_keys
|
||||
WHERE id = ${storageKeyId}
|
||||
FOR UPDATE
|
||||
`
|
||||
if (!lockedKeys || lockedKeys.length === 0 || lockedKeys[0].isArchived) {
|
||||
return
|
||||
}
|
||||
|
||||
// If an auto-archive policy was specified, re-verify under lock that all sibling assets meet the age policy
|
||||
if (options?.autoArchivePolicy) {
|
||||
const policy = options.autoArchivePolicy
|
||||
const ineligible = await tx.asset.findFirst({
|
||||
where: {
|
||||
storageKeyId,
|
||||
isDeleted: false,
|
||||
OR: [
|
||||
{ type: { not: AssetType.file } },
|
||||
{ status: { not: AssetStatus.processed } },
|
||||
...(policy.method === 'not_updated'
|
||||
? [{ updatedAt: { gt: policy.cutoffDate } }]
|
||||
: [
|
||||
{
|
||||
OR: [
|
||||
{ createdAt: { gt: policy.cutoffDate } },
|
||||
{ updatedAt: { gt: policy.restoreCutoffDate } },
|
||||
],
|
||||
},
|
||||
]),
|
||||
],
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
if (ineligible) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Atomic Filter + Update: only files that are currently 'processed' transition to 'archiving'
|
||||
const updatedAssets = await tx.$queryRaw<Array<{ id: string }>>`
|
||||
UPDATE assets
|
||||
SET status = 'archiving'::"AssetStatus", updated_at = NOW()
|
||||
WHERE storage_key_id = ${storageKeyId}
|
||||
AND type = 'file'::"AssetType"
|
||||
AND status = 'processed'::"AssetStatus"
|
||||
AND is_deleted = false
|
||||
RETURNING id
|
||||
`
|
||||
|
||||
if (!updatedAssets || updatedAssets.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const row of updatedAssets) {
|
||||
archivedAssetIdsSet.add(row.id)
|
||||
}
|
||||
|
||||
const existingTask = await tx.workflowTask.findFirst({
|
||||
where: {
|
||||
assetId: { in: filesForKey.map((f) => f.id) },
|
||||
type: WorkflowTaskType.archive_asset,
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
if (existingTask) {
|
||||
taskIds.push(existingTask.id)
|
||||
} else {
|
||||
const createdTask = await tx.workflowTask.create({
|
||||
data: {
|
||||
assetId: primaryFile.id,
|
||||
type: WorkflowTaskType.archive_asset,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
teamId: primaryFile.project!.teamId,
|
||||
projectId: primaryFile.projectId!,
|
||||
payload: {
|
||||
projectId: primaryFile.projectId!,
|
||||
archive: { storageKeyId },
|
||||
},
|
||||
},
|
||||
})
|
||||
taskIds.push(createdTask.id)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
archivedAssetIds: Array.from(archivedAssetIdsSet),
|
||||
taskIds,
|
||||
skippedCount,
|
||||
}
|
||||
}
|
||||
|
||||
async restoreArchivedAssets(assetIds: string[]): Promise<RestoreArchivedAssetsResponse> {
|
||||
if (!isArchiveStorageEnabled()) {
|
||||
throw new HTTPException(400, { message: 'Archive storage is not configured' })
|
||||
}
|
||||
if (assetIds.length === 0) {
|
||||
return { restoredAssetIds: [], taskIds: [], skippedCount: 0 }
|
||||
}
|
||||
|
||||
const candidateFiles = await this.resolveCandidateFileAssetsForArchiveOrRestore(assetIds)
|
||||
if (candidateFiles.length === 0) {
|
||||
return { restoredAssetIds: [], taskIds: [], skippedCount: 0 }
|
||||
}
|
||||
|
||||
const candidateStorageKeyIds = Array.from(
|
||||
new Set(candidateFiles.map((f) => f.storageKeyId).filter((id): id is string => !!id)),
|
||||
)
|
||||
|
||||
const allFilesForKeys = await this.prismaClient.asset.findMany({
|
||||
where: {
|
||||
storageKeyId: { in: candidateStorageKeyIds },
|
||||
isDeleted: false,
|
||||
type: AssetType.file,
|
||||
status: { notIn: [AssetStatus.trashed, AssetStatus.pending_purge] },
|
||||
},
|
||||
select: { id: true, storageKeyId: true, status: true },
|
||||
})
|
||||
|
||||
const storageKeysWithArchivingSibling = new Set<string>()
|
||||
for (const f of allFilesForKeys) {
|
||||
if (!f.storageKeyId) continue
|
||||
if (f.status === AssetStatus.archiving) {
|
||||
storageKeysWithArchivingSibling.add(f.storageKeyId)
|
||||
}
|
||||
}
|
||||
|
||||
const eligibleByStorageKey = new Map<string, typeof candidateFiles>()
|
||||
let skippedCount = 0
|
||||
let alreadyRestoringOrHotCount = 0
|
||||
let currentlyArchivingCount = 0
|
||||
|
||||
for (const file of candidateFiles) {
|
||||
if (!file.storageKeyId || !file.storageKey || !file.projectId || !file.project?.teamId) {
|
||||
skippedCount++
|
||||
continue
|
||||
}
|
||||
if (
|
||||
file.status === AssetStatus.archiving ||
|
||||
storageKeysWithArchivingSibling.has(file.storageKeyId)
|
||||
) {
|
||||
skippedCount++
|
||||
currentlyArchivingCount++
|
||||
continue
|
||||
}
|
||||
if (file.status === AssetStatus.restoring) {
|
||||
skippedCount++
|
||||
alreadyRestoringOrHotCount++
|
||||
continue
|
||||
}
|
||||
if (file.status !== AssetStatus.archived && !file.storageKey.isArchived) {
|
||||
skippedCount++
|
||||
alreadyRestoringOrHotCount++
|
||||
continue
|
||||
}
|
||||
|
||||
const list = eligibleByStorageKey.get(file.storageKeyId) || []
|
||||
list.push(file)
|
||||
eligibleByStorageKey.set(file.storageKeyId, list)
|
||||
}
|
||||
|
||||
if (eligibleByStorageKey.size === 0) {
|
||||
if (alreadyRestoringOrHotCount === 0 && currentlyArchivingCount > 0) {
|
||||
throw new HTTPException(409, {
|
||||
message: 'Cannot restore asset while it is currently archiving',
|
||||
})
|
||||
}
|
||||
return { restoredAssetIds: [], taskIds: [], skippedCount }
|
||||
}
|
||||
|
||||
const restoredAssetIdsSet = new Set<string>()
|
||||
const taskIds: string[] = []
|
||||
|
||||
for (const [storageKeyId, filesForKey] of eligibleByStorageKey.entries()) {
|
||||
const primaryFile = filesForKey[0]
|
||||
await this.prismaClient.$transaction(async (tx) => {
|
||||
const sharedAssets = await tx.asset.findMany({
|
||||
where: {
|
||||
storageKeyId,
|
||||
isDeleted: false,
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.archived,
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
const sharedAssetIds = sharedAssets.map((a) => a.id)
|
||||
|
||||
if (sharedAssetIds.length > 0) {
|
||||
await tx.asset.updateMany({
|
||||
where: { id: { in: sharedAssetIds } },
|
||||
data: { status: AssetStatus.restoring },
|
||||
})
|
||||
|
||||
for (const id of sharedAssetIds) {
|
||||
restoredAssetIdsSet.add(id)
|
||||
}
|
||||
}
|
||||
|
||||
const existingTask = await tx.workflowTask.findFirst({
|
||||
where: {
|
||||
assetId: { in: filesForKey.map((f) => f.id) },
|
||||
type: WorkflowTaskType.restore_asset,
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
if (existingTask) {
|
||||
taskIds.push(existingTask.id)
|
||||
} else {
|
||||
const createdTask = await tx.workflowTask.create({
|
||||
data: {
|
||||
assetId: primaryFile.id,
|
||||
type: WorkflowTaskType.restore_asset,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
teamId: primaryFile.project!.teamId,
|
||||
projectId: primaryFile.projectId!,
|
||||
payload: {
|
||||
projectId: primaryFile.projectId!,
|
||||
archive: { storageKeyId },
|
||||
},
|
||||
},
|
||||
})
|
||||
taskIds.push(createdTask.id)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
restoredAssetIds: Array.from(restoredAssetIdsSet),
|
||||
taskIds,
|
||||
skippedCount,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const assetArchiveService = new AssetArchiveService()
|
||||
|
||||
/**
|
||||
* Reverts assets referencing a StorageKey back to their baseline status (archived vs processed)
|
||||
* when an archive or restore task fails or recovers.
|
||||
*/
|
||||
export async function revertArchiveStateForStorageKey(
|
||||
storageKeyId: string,
|
||||
client: PrismaClient | Prisma.TransactionClient = prisma,
|
||||
): Promise<void> {
|
||||
const sk = await client.storageKey.findUnique({
|
||||
where: { id: storageKeyId },
|
||||
select: { isArchived: true },
|
||||
})
|
||||
if (!sk) return
|
||||
|
||||
const fallbackStatus = sk.isArchived ? AssetStatus.archived : AssetStatus.processed
|
||||
await client.asset.updateMany({
|
||||
where: {
|
||||
storageKeyId,
|
||||
isDeleted: false,
|
||||
deletedAt: null,
|
||||
type: AssetType.file,
|
||||
status: { in: [AssetStatus.archiving, AssetStatus.restoring] },
|
||||
},
|
||||
data: { status: fallbackStatus },
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures an asset's storage key is not archived before allowing direct download of the original raw file.
|
||||
*/
|
||||
export async function assertStorageKeyNotArchived(
|
||||
key: string,
|
||||
client: PrismaClient | Prisma.TransactionClient = prisma,
|
||||
): Promise<void> {
|
||||
const sk = await client.storageKey.findUnique({
|
||||
where: { key },
|
||||
include: {
|
||||
assets: {
|
||||
where: { isDeleted: false },
|
||||
select: { status: true },
|
||||
},
|
||||
},
|
||||
})
|
||||
if (
|
||||
sk &&
|
||||
(sk.isArchived ||
|
||||
sk.assets.some(
|
||||
(a) =>
|
||||
a.status === AssetStatus.archiving ||
|
||||
a.status === AssetStatus.archived ||
|
||||
a.status === AssetStatus.restoring,
|
||||
))
|
||||
) {
|
||||
throw new Error(
|
||||
`Original file for storage key "${key}" is archived. Restore the asset first or download its proxy using assetIds with proxy: true.`,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,8 @@ import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
|
||||
import { AssetType, AssetStatus, Prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { s3Service, getStorageBackend } from '@shumai/core/src/s3/s3'
|
||||
import * as archiveStorageModule from '@shumai/core/src/s3/archive-storage'
|
||||
import { uploadService } from '@shumai/core/src/upload/upload'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { AssetService } from './asset'
|
||||
@@ -20,7 +21,9 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
deleteObject: vi.fn().mockResolvedValue(1),
|
||||
deletePrefix: vi.fn().mockResolvedValue(1),
|
||||
copyObject: vi.fn().mockResolvedValue(undefined),
|
||||
getStoragePath: vi.fn((bucket: string, key: string) => `/mock/data/${bucket}/${key}`),
|
||||
},
|
||||
getStorageBackend: vi.fn().mockReturnValue('local'),
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/s3/preview-converter', async (importOriginal) => {
|
||||
@@ -3010,7 +3013,7 @@ describe('AssetService — natural sort by name', () => {
|
||||
// 4. file1.txt (from symlinkToFile target resolution, deduplicated)
|
||||
// 5. version2.txt (from symlinkToStack target resolution, deduplicated)
|
||||
|
||||
const fileNames = result.map((r) => r.name)
|
||||
const fileNames = result.files.map((r) => r.name)
|
||||
expect(fileNames).toHaveLength(3) // 3 unique files
|
||||
expect(fileNames).toContain('file1.txt')
|
||||
expect(fileNames).toContain('file2.txt')
|
||||
@@ -3021,7 +3024,7 @@ describe('AssetService — natural sort by name', () => {
|
||||
expect(fileNames).not.toContain('stack')
|
||||
|
||||
// Check urls are returned correctly (mock s3 presign resolves to 'http://mock-s3-url')
|
||||
for (const item of result) {
|
||||
for (const item of result.files) {
|
||||
expect(item.url).toBe('http://mock-s3-url')
|
||||
}
|
||||
|
||||
@@ -3070,9 +3073,251 @@ describe('AssetService — natural sort by name', () => {
|
||||
})
|
||||
|
||||
const result = await assetService.getDownloadLinks([symlink.id])
|
||||
expect(result).toHaveLength(1)
|
||||
expect(result[0].name).toBe('file1.txt')
|
||||
expect(result[0].id).toBe(file1.id)
|
||||
expect(result.files).toHaveLength(1)
|
||||
expect(result.files[0].name).toBe('file1.txt')
|
||||
expect(result.files[0].id).toBe(file1.id)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getStoragePaths', () => {
|
||||
it('returns empty array for empty inputs', async () => {
|
||||
const result = await assetService.getStoragePaths([])
|
||||
expect(result).toEqual({ paths: [] })
|
||||
})
|
||||
|
||||
it('resolves files, symlinks, and version stacks while skipping folders', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
name: 'StoragePathUser',
|
||||
email: `storage-path-${Date.now()}@example.com`,
|
||||
},
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 'StoragePathTeam' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'StoragePathProject', teamId: team.id },
|
||||
})
|
||||
|
||||
const folder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'MyFolder',
|
||||
type: AssetType.folder,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.processed,
|
||||
},
|
||||
})
|
||||
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'clip.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.processed,
|
||||
storageKey: { create: { key: 'files/ulid1/clip.mp4' } },
|
||||
},
|
||||
})
|
||||
|
||||
const symlink = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'clip-symlink',
|
||||
type: AssetType.symlink,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.uploaded,
|
||||
target: { connect: { id: file.id } },
|
||||
},
|
||||
})
|
||||
|
||||
const stack = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'stack.png',
|
||||
type: AssetType.version_stack,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.processed,
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'stack_v1.png',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
parent: { connect: { id: stack.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.processed,
|
||||
sortIndex: 'b',
|
||||
storageKey: { create: { key: 'files/ulid2/stack_v1.png' } },
|
||||
},
|
||||
})
|
||||
|
||||
const latestChild = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'stack_v2.png',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
parent: { connect: { id: stack.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.processed,
|
||||
sortIndex: 'a',
|
||||
storageKey: { create: { key: 'files/ulid3/stack_v2.png' } },
|
||||
},
|
||||
})
|
||||
|
||||
const result = await assetService.getStoragePaths([folder.id, file.id, symlink.id, stack.id])
|
||||
|
||||
// folder should be skipped, symlink deduplicated to file, stack resolved to latest child
|
||||
expect(result.paths).toHaveLength(2)
|
||||
const names = result.paths.map((p) => p.name)
|
||||
expect(names).toContain('clip.mp4')
|
||||
expect(names).toContain('stack_v2.png')
|
||||
|
||||
const clipPath = result.paths.find((p) => p.name === 'clip.mp4')!
|
||||
expect(clipPath.path).toContain('files/ulid1/clip.mp4')
|
||||
expect(clipPath.isArchived).toBe(false)
|
||||
|
||||
const stackPath = result.paths.find((p) => p.name === 'stack_v2.png')!
|
||||
expect(stackPath.id).toBe(latestChild.id)
|
||||
expect(stackPath.path).toContain('files/ulid3/stack_v2.png')
|
||||
})
|
||||
|
||||
it('returns S3 path when main storage backend is s3', async () => {
|
||||
vi.mocked(getStorageBackend).mockReturnValueOnce('s3')
|
||||
vi.mocked(s3Service.getStoragePath).mockReturnValueOnce('s3://shumai/files/test/s3-file.mp4')
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'P-S3', team: { create: { name: 'T-S3' } } },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'U-S3', email: `s3-${Date.now()}@example.com` },
|
||||
})
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 's3-file.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.processed,
|
||||
storageKey: { create: { key: 'files/test/s3-file.mp4' } },
|
||||
},
|
||||
})
|
||||
|
||||
const res = await assetService.getStoragePaths([file.id])
|
||||
expect(res.paths).toEqual([
|
||||
{
|
||||
id: file.id,
|
||||
name: 's3-file.mp4',
|
||||
path: 's3://shumai/files/test/s3-file.mp4',
|
||||
backend: 's3',
|
||||
isArchived: false,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('returns archive S3 path when asset is archived and archive backend is s3', async () => {
|
||||
vi.spyOn(archiveStorageModule, 'getArchiveStoragePath').mockReturnValueOnce({
|
||||
path: 's3://my-archive-bucket/files/archived/clip.mp4',
|
||||
backend: 's3',
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'P-Archive-S3', team: { create: { name: 'T-Archive-S3' } } },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'U-Arch', email: `arch-${Date.now()}@example.com` },
|
||||
})
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'clip.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.archived,
|
||||
storageKey: { create: { key: 'files/archived/clip.mp4', isArchived: true } },
|
||||
},
|
||||
})
|
||||
|
||||
const res = await assetService.getStoragePaths([file.id])
|
||||
expect(res.paths).toEqual([
|
||||
{
|
||||
id: file.id,
|
||||
name: 'clip.mp4',
|
||||
path: 's3://my-archive-bucket/files/archived/clip.mp4',
|
||||
backend: 's3',
|
||||
isArchived: true,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('returns archive local path when asset is archived and archive backend is local', async () => {
|
||||
vi.spyOn(archiveStorageModule, 'getArchiveStoragePath').mockReturnValueOnce({
|
||||
path: '/mnt/archive/files/archived/clip.mp4',
|
||||
backend: 'local',
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'P-Archive-Local', team: { create: { name: 'T-Archive-Local' } } },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'U-ArchLocal', email: `archloc-${Date.now()}@example.com` },
|
||||
})
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'clip.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.archived,
|
||||
storageKey: { create: { key: 'files/archived/clip.mp4', isArchived: true } },
|
||||
},
|
||||
})
|
||||
|
||||
const res = await assetService.getStoragePaths([file.id])
|
||||
expect(res.paths).toEqual([
|
||||
{
|
||||
id: file.id,
|
||||
name: 'clip.mp4',
|
||||
path: '/mnt/archive/files/archived/clip.mp4',
|
||||
backend: 'local',
|
||||
isArchived: true,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('marks isArchived: true with main storage path when archive storage is disabled', async () => {
|
||||
vi.spyOn(archiveStorageModule, 'getArchiveStoragePath').mockReturnValueOnce(null)
|
||||
vi.mocked(s3Service.getStoragePath).mockReturnValueOnce(
|
||||
'/mock/data/shumai/files/archived/clip.mp4',
|
||||
)
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'P-DisabledArchive', team: { create: { name: 'T-DisabledArchive' } } },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'U-DisArch', email: `disarch-${Date.now()}@example.com` },
|
||||
})
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'clip.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: project.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
status: AssetStatus.archived,
|
||||
storageKey: { create: { key: 'files/archived/clip.mp4', isArchived: true } },
|
||||
},
|
||||
})
|
||||
|
||||
const res = await assetService.getStoragePaths([file.id])
|
||||
expect(res.paths).toEqual([
|
||||
{
|
||||
id: file.id,
|
||||
name: 'clip.mp4',
|
||||
path: '/mock/data/shumai/files/archived/clip.mp4',
|
||||
backend: 'local',
|
||||
isArchived: true,
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { cancelWorkflowTask, prisma } from '@shumai/db'
|
||||
import {
|
||||
AncestorFolder,
|
||||
ArchiveAssetsResponse,
|
||||
AssetInfo,
|
||||
AttachmentInfo,
|
||||
ChildPreview,
|
||||
@@ -14,6 +15,7 @@ import {
|
||||
ListChildrenRequest,
|
||||
PreviewInfo,
|
||||
ReparentAssetsRequest,
|
||||
RestoreArchivedAssetsResponse,
|
||||
UpdateAssetNameRequest,
|
||||
UpdateAssetOrderRequest,
|
||||
AssetUserInfo,
|
||||
@@ -21,6 +23,8 @@ import {
|
||||
CommentExportFormat,
|
||||
PreviewFormat,
|
||||
TRASH_RETENTION_DAYS,
|
||||
GetStoragePathsResponse,
|
||||
StoragePathItem,
|
||||
} from '@shumai/dtos'
|
||||
import {
|
||||
exportCommentsToFormat,
|
||||
@@ -41,7 +45,13 @@ import {
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { PaginatedData, paginateQuery, PaginationParams } from '@shumai/core/src/pagination'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { s3Service, getStorageBackend } from '@shumai/core/src/s3/s3'
|
||||
import {
|
||||
getArchiveStorageService,
|
||||
isArchiveStorageEnabled,
|
||||
getArchiveStoragePath,
|
||||
} from '@shumai/core/src/s3/archive-storage'
|
||||
import { assetArchiveService } from './asset-archive'
|
||||
import {
|
||||
ensureJpegInStorage,
|
||||
getJpegKeyForWebp,
|
||||
@@ -409,7 +419,7 @@ export class AssetService {
|
||||
|
||||
const assetsToCopy = await tx.asset.findMany({
|
||||
where: { id: { in: req.assetIds } },
|
||||
include: { project: { include: { team: true } } },
|
||||
include: { project: { include: { team: true } }, storageKey: true },
|
||||
})
|
||||
|
||||
if (assetsToCopy.length !== req.assetIds.length) {
|
||||
@@ -488,6 +498,17 @@ export class AssetService {
|
||||
creatorId?: string,
|
||||
withComments: boolean = false,
|
||||
): Promise<Asset> {
|
||||
if (asset.status === AssetStatus.archiving || asset.status === AssetStatus.restoring) {
|
||||
throw new HTTPException(409, {
|
||||
message: 'Cannot copy an asset while it is being archived or restored',
|
||||
})
|
||||
}
|
||||
|
||||
const copiedStatus =
|
||||
asset.type === AssetType.file && asset.storageKey?.isArchived
|
||||
? AssetStatus.archived
|
||||
: asset.status
|
||||
|
||||
const newAsset = await tx.asset.create({
|
||||
data: {
|
||||
name: asset.name,
|
||||
@@ -496,7 +517,7 @@ export class AssetService {
|
||||
mediaType: asset.mediaType,
|
||||
fileCount: asset.fileCount,
|
||||
sizeByte: asset.sizeByte,
|
||||
status: asset.status,
|
||||
status: copiedStatus,
|
||||
transcodeTaskId: asset.transcodeTaskId,
|
||||
media: asset.media || undefined,
|
||||
isDeleted: asset.isDeleted,
|
||||
@@ -515,9 +536,10 @@ export class AssetService {
|
||||
await this.copyComments(tx, asset.id, newAsset.id, projectId, creatorId)
|
||||
}
|
||||
|
||||
if (asset.type === AssetType.folder) {
|
||||
if (asset.type === AssetType.folder || asset.type === AssetType.version_stack) {
|
||||
const children = await tx.asset.findMany({
|
||||
where: { parentId: asset.id },
|
||||
include: { storageKey: true },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
})
|
||||
for (const child of children) {
|
||||
@@ -1229,6 +1251,75 @@ export class AssetService {
|
||||
return currentId
|
||||
}
|
||||
|
||||
async resolveLatestVersionIds(assetIds: string[]): Promise<string[]> {
|
||||
if (assetIds.length === 0) return []
|
||||
|
||||
const initialAssets = await this.prismaClient.asset.findMany({
|
||||
where: { id: { in: assetIds }, isDeleted: false },
|
||||
select: { id: true, type: true, targetId: true },
|
||||
})
|
||||
const assetMap = new Map(initialAssets.map((a) => [a.id, a]))
|
||||
|
||||
// Find symlinks to resolve
|
||||
const symlinks = initialAssets.filter((a) => a.type === AssetType.symlink && a.targetId)
|
||||
const targetIds = Array.from(new Set(symlinks.map((s) => s.targetId!)))
|
||||
|
||||
const symlinkTargetMap = new Map<string, { id: string; type: AssetType }>()
|
||||
if (targetIds.length > 0) {
|
||||
const targets = await this.prismaClient.asset.findMany({
|
||||
where: { id: { in: targetIds }, isDeleted: false },
|
||||
select: { id: true, type: true },
|
||||
})
|
||||
for (const t of targets) {
|
||||
symlinkTargetMap.set(t.id, t)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine current ID and type for each input ID
|
||||
const intermediate = assetIds.map((id) => {
|
||||
const asset = assetMap.get(id)
|
||||
if (!asset) return { originalId: id, currentId: id, type: null }
|
||||
if (asset.type === AssetType.symlink && asset.targetId) {
|
||||
const target = symlinkTargetMap.get(asset.targetId)
|
||||
if (target) {
|
||||
return { originalId: id, currentId: target.id, type: target.type }
|
||||
}
|
||||
}
|
||||
return { originalId: id, currentId: asset.id, type: asset.type }
|
||||
})
|
||||
|
||||
// Find version stacks to resolve to their latest child
|
||||
const versionStackIds = Array.from(
|
||||
new Set(
|
||||
intermediate
|
||||
.filter((item) => item.type === AssetType.version_stack)
|
||||
.map((item) => item.currentId),
|
||||
),
|
||||
)
|
||||
|
||||
const latestChildByParent = new Map<string, string>()
|
||||
if (versionStackIds.length > 0) {
|
||||
const children = await this.prismaClient.asset.findMany({
|
||||
where: { parentId: { in: versionStackIds }, isDeleted: false },
|
||||
select: { id: true, parentId: true, sortIndex: true },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
})
|
||||
for (const child of children) {
|
||||
if (child.parentId && !latestChildByParent.has(child.parentId)) {
|
||||
latestChildByParent.set(child.parentId, child.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return intermediate.map((item) => {
|
||||
if (item.type === AssetType.version_stack) {
|
||||
const childId = latestChildByParent.get(item.currentId)
|
||||
if (childId) return childId
|
||||
}
|
||||
return item.currentId
|
||||
})
|
||||
}
|
||||
|
||||
async updateAssetName(req: UpdateAssetNameRequest): Promise<AssetInfo> {
|
||||
const existing = await this.prismaClient.asset.findUnique({
|
||||
where: { id: req.id },
|
||||
@@ -1605,7 +1696,9 @@ export class AssetService {
|
||||
|
||||
// 1. Find storage keys with no associated assets and older than 24 hours
|
||||
// using FOR UPDATE SKIP LOCKED to ensure multiple servers don't pick up the same keys.
|
||||
const lockedKeys = await this.prismaClient.$queryRaw<{ id: string; key: string }[]>`
|
||||
const lockedKeys = await this.prismaClient.$queryRaw<
|
||||
{ id: string; key: string; isArchived: boolean }[]
|
||||
>`
|
||||
WITH to_purge AS (
|
||||
SELECT id FROM storage_keys sk
|
||||
WHERE NOT EXISTS (SELECT 1 FROM assets a WHERE a.storage_key_id = sk.id)
|
||||
@@ -1616,17 +1709,30 @@ export class AssetService {
|
||||
)
|
||||
UPDATE storage_keys SET status = 'purging', updated_at = NOW()
|
||||
WHERE id IN (SELECT id FROM to_purge)
|
||||
RETURNING id, key;
|
||||
RETURNING id, key, is_archived AS "isArchived";
|
||||
`
|
||||
|
||||
if (lockedKeys.length === 0) return
|
||||
|
||||
let physicalFilesDeleted = 0
|
||||
let purgedCount = 0
|
||||
const archiveEnabled = isArchiveStorageEnabled()
|
||||
|
||||
for (const sk of lockedKeys) {
|
||||
try {
|
||||
// Physically delete from S3
|
||||
if (sk.isArchived && !archiveEnabled) {
|
||||
logger.warn(
|
||||
{ storageKeyId: sk.id, key: sk.key },
|
||||
'Skipping purge of archived storage key because archive storage is not configured',
|
||||
)
|
||||
await this.prismaClient.storageKey.update({
|
||||
where: { id: sk.id },
|
||||
data: { status: 'active' },
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Physically delete proxies (and unarchived original) from main storage
|
||||
const parts = sk.key.split('/')
|
||||
if (parts.length > 2) {
|
||||
const prefix = parts.slice(0, parts.length - 1).join('/') + '/'
|
||||
@@ -1637,6 +1743,15 @@ export class AssetService {
|
||||
physicalFilesDeleted += count
|
||||
}
|
||||
|
||||
// Also delete original raw file from archive storage when archive storage is separate
|
||||
if (archiveEnabled) {
|
||||
const archiveStorage = getArchiveStorageService()
|
||||
if (!archiveStorage.isSameS3Target) {
|
||||
const archiveDeleted = await archiveStorage.deleteObject(sk.key)
|
||||
physicalFilesDeleted += archiveDeleted
|
||||
}
|
||||
}
|
||||
|
||||
// Delete from database
|
||||
await this.prismaClient.storageKey.delete({
|
||||
where: { id: sk.id },
|
||||
@@ -1663,7 +1778,7 @@ export class AssetService {
|
||||
await this.prismaClient.$transaction(async (tx) => {
|
||||
const a = await tx.asset.findUnique({
|
||||
where: { id },
|
||||
include: { project: { include: { team: true } } },
|
||||
include: { project: { include: { team: true } }, storageKey: true },
|
||||
})
|
||||
if (!a) throw new Error('Asset not found')
|
||||
if (!a.parentId) throw new Error('Asset has no parent')
|
||||
@@ -1687,7 +1802,25 @@ export class AssetService {
|
||||
if (descendantIds.length > 0) {
|
||||
await tx.asset.updateMany({
|
||||
where: { id: { in: descendantIds } },
|
||||
data: { isDeleted: false },
|
||||
data: { isDeleted: false, deletedAt: null },
|
||||
})
|
||||
await tx.asset.updateMany({
|
||||
where: {
|
||||
id: { in: descendantIds },
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.trashed,
|
||||
storageKey: { isArchived: true },
|
||||
},
|
||||
data: { status: AssetStatus.archived },
|
||||
})
|
||||
await tx.asset.updateMany({
|
||||
where: {
|
||||
id: { in: descendantIds },
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.trashed,
|
||||
storageKey: { isArchived: false },
|
||||
},
|
||||
data: { status: AssetStatus.processed },
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1724,6 +1857,16 @@ export class AssetService {
|
||||
}
|
||||
}
|
||||
|
||||
const activeArchiveTask = await tx.workflowTask.findFirst({
|
||||
where: {
|
||||
assetId: a.id,
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
type: {
|
||||
in: [WorkflowTaskType.archive_asset, WorkflowTaskType.restore_asset],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const activeTask = await tx.workflowTask.findFirst({
|
||||
where: {
|
||||
assetId: a.id,
|
||||
@@ -1743,11 +1886,22 @@ export class AssetService {
|
||||
},
|
||||
})
|
||||
|
||||
let restoredStatus: AssetStatus = AssetStatus.processed
|
||||
if (activeArchiveTask?.type === WorkflowTaskType.archive_asset) {
|
||||
restoredStatus = AssetStatus.archiving
|
||||
} else if (activeArchiveTask?.type === WorkflowTaskType.restore_asset) {
|
||||
restoredStatus = AssetStatus.restoring
|
||||
} else if (activeTask) {
|
||||
restoredStatus = AssetStatus.processing
|
||||
} else if (a.type === AssetType.file && a.storageKey?.isArchived) {
|
||||
restoredStatus = AssetStatus.archived
|
||||
}
|
||||
|
||||
await tx.asset.update({
|
||||
where: { id: a.id },
|
||||
data: {
|
||||
isDeleted: false,
|
||||
status: activeTask ? AssetStatus.processing : AssetStatus.processed,
|
||||
status: restoredStatus,
|
||||
deletedAt: null,
|
||||
},
|
||||
})
|
||||
@@ -1805,7 +1959,12 @@ export class AssetService {
|
||||
const attachmentAsset = await tx.asset.findUnique({
|
||||
where: { id: attachmentId },
|
||||
})
|
||||
if (attachmentAsset) {
|
||||
if (
|
||||
attachmentAsset &&
|
||||
(attachmentAsset.type === AssetType.attachment ||
|
||||
attachmentAsset.type === AssetType.file) &&
|
||||
attachmentAsset.projectId === a.projectId
|
||||
) {
|
||||
await tx.assetCommentAttachment.create({
|
||||
data: {
|
||||
assetId: attachmentAsset.id,
|
||||
@@ -2352,6 +2511,7 @@ export class AssetService {
|
||||
current: i === 0,
|
||||
id: v.id,
|
||||
name: v.name,
|
||||
status: v.status,
|
||||
previewUrl: preview?.thumbnailUrl || null,
|
||||
createdAt: v.createdAt.toISOString(),
|
||||
creator: v.creator
|
||||
@@ -2445,7 +2605,15 @@ export class AssetService {
|
||||
}
|
||||
}
|
||||
|
||||
let media = latestVersion.media as PrismaJson.MediaInfo | null
|
||||
const isOriginalUnavailable =
|
||||
latestVersion.status === AssetStatus.archiving ||
|
||||
latestVersion.status === AssetStatus.archived ||
|
||||
latestVersion.status === AssetStatus.restoring ||
|
||||
Boolean(latestVersion.storageKey?.isArchived)
|
||||
|
||||
let media = latestVersion.media
|
||||
? (structuredClone(latestVersion.media) as PrismaJson.MediaInfo)
|
||||
: null
|
||||
if (media && media.videoPreview?.key) {
|
||||
media.videoPreview.url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
@@ -2454,11 +2622,15 @@ export class AssetService {
|
||||
)
|
||||
}
|
||||
if (media && media.pdfTranscode?.key) {
|
||||
media.pdfTranscode.url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
media.pdfTranscode.key,
|
||||
'GET',
|
||||
)
|
||||
if (isOriginalUnavailable && media.pdfTranscode.key === latestVersion.storageKey?.key) {
|
||||
delete media.pdfTranscode.url
|
||||
} else {
|
||||
media.pdfTranscode.url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
media.pdfTranscode.key,
|
||||
'GET',
|
||||
)
|
||||
}
|
||||
}
|
||||
if (media && media.isHls && media.hls?.key) {
|
||||
media.hls.url = `/api/files/${latestVersion.id}/m3u8/master.m3u8`
|
||||
@@ -2523,6 +2695,7 @@ export class AssetService {
|
||||
id: string
|
||||
version: number
|
||||
name: string
|
||||
status: AssetStatus
|
||||
previewUrl: string | null
|
||||
createdAt: string
|
||||
creator: { id: string; name: string | null; image?: string | null } | null
|
||||
@@ -2546,6 +2719,7 @@ export class AssetService {
|
||||
id: v.id,
|
||||
version: versions.length - i,
|
||||
name: v.name,
|
||||
status: v.status,
|
||||
previewUrl: preview?.thumbnailUrl || null,
|
||||
createdAt: v.createdAt.toISOString(),
|
||||
creator: v.creator
|
||||
@@ -2605,8 +2779,11 @@ export class AssetService {
|
||||
return Array.from(projectIds)
|
||||
}
|
||||
|
||||
async getDownloadLinks(ids: string[]): Promise<Array<{ id: string; name: string; url: string }>> {
|
||||
if (ids.length === 0) return []
|
||||
async getDownloadLinks(ids: string[]): Promise<{
|
||||
files: Array<{ id: string; name: string; url: string }>
|
||||
skippedArchivedCount: number
|
||||
}> {
|
||||
if (ids.length === 0) return { files: [], skippedArchivedCount: 0 }
|
||||
|
||||
// Resolve starting IDs (dereference top-level symlinks)
|
||||
const assets = await this.prismaClient.asset.findMany({
|
||||
@@ -2625,13 +2802,14 @@ export class AssetService {
|
||||
return asset.id
|
||||
})
|
||||
|
||||
if (startingIds.length === 0) return []
|
||||
if (startingIds.length === 0) return { files: [], skippedArchivedCount: 0 }
|
||||
|
||||
const descendants = await this.prismaClient.$queryRaw<
|
||||
Array<{
|
||||
id: string
|
||||
name: string
|
||||
type: string
|
||||
status: string
|
||||
parentId: string | null
|
||||
storageKeyId: string | null
|
||||
sortIndex: string | null
|
||||
@@ -2639,16 +2817,16 @@ export class AssetService {
|
||||
}>
|
||||
>`
|
||||
WITH RECURSIVE descendant AS (
|
||||
SELECT id, name, type, parent_id, storage_key_id, sort_index, target_id, is_deleted
|
||||
SELECT id, name, type, status::text AS status, parent_id, storage_key_id, sort_index, target_id, is_deleted
|
||||
FROM assets
|
||||
WHERE id IN (${Prisma.join(startingIds)}) AND is_deleted = false
|
||||
UNION ALL
|
||||
SELECT a.id, a.name, a.type, a.parent_id, a.storage_key_id, a.sort_index, a.target_id, a.is_deleted
|
||||
SELECT a.id, a.name, a.type, a.status::text AS status, a.parent_id, a.storage_key_id, a.sort_index, a.target_id, a.is_deleted
|
||||
FROM assets a
|
||||
INNER JOIN descendant d ON a.parent_id = d.id
|
||||
WHERE a.is_deleted = false
|
||||
)
|
||||
SELECT id, name, type, parent_id AS "parentId", storage_key_id AS "storageKeyId", sort_index AS "sortIndex", target_id AS "targetId" FROM descendant;
|
||||
SELECT id, name, type, status, parent_id AS "parentId", storage_key_id AS "storageKeyId", sort_index AS "sortIndex", target_id AS "targetId" FROM descendant;
|
||||
`
|
||||
|
||||
// Find any symlink in the descendants and resolve its target.
|
||||
@@ -2657,6 +2835,7 @@ export class AssetService {
|
||||
id: string
|
||||
name: string
|
||||
type: string
|
||||
status: string
|
||||
parentId: string | null
|
||||
storageKeyId: string | null
|
||||
sortIndex: string | null
|
||||
@@ -2673,6 +2852,7 @@ export class AssetService {
|
||||
id: t.id,
|
||||
name: t.name,
|
||||
type: t.type,
|
||||
status: t.status,
|
||||
parentId: t.parentId,
|
||||
storageKeyId: t.storageKeyId,
|
||||
sortIndex: t.sortIndex,
|
||||
@@ -2737,16 +2917,29 @@ export class AssetService {
|
||||
const storageKeys = await this.prismaClient.storageKey.findMany({
|
||||
where: { id: { in: storageKeyIds } },
|
||||
})
|
||||
const storageKeyMap = new Map(storageKeys.map((k) => [k.id, k.key]))
|
||||
const storageKeyMap = new Map(storageKeys.map((k) => [k.id, k]))
|
||||
|
||||
let skippedArchivedCount = 0
|
||||
|
||||
const downloadLinks = await Promise.all(
|
||||
deduplicatedFiles.map(async (file) => {
|
||||
const key = file.storageKeyId ? storageKeyMap.get(file.storageKeyId) : null
|
||||
if (!key) return null
|
||||
const sk = file.storageKeyId ? storageKeyMap.get(file.storageKeyId) : null
|
||||
if (!sk?.key) return null
|
||||
|
||||
const isArchivedOrTransitioning =
|
||||
file.status === AssetStatus.archiving ||
|
||||
file.status === AssetStatus.archived ||
|
||||
file.status === AssetStatus.restoring ||
|
||||
sk.isArchived
|
||||
|
||||
if (isArchivedOrTransitioning) {
|
||||
skippedArchivedCount++
|
||||
return null
|
||||
}
|
||||
|
||||
const url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
key,
|
||||
sk.key,
|
||||
'GET',
|
||||
true,
|
||||
file.name,
|
||||
@@ -2760,7 +2953,79 @@ export class AssetService {
|
||||
}),
|
||||
)
|
||||
|
||||
return downloadLinks.filter((link): link is { id: string; name: string; url: string } => !!link)
|
||||
const filteredLinks = downloadLinks.filter(
|
||||
(link): link is { id: string; name: string; url: string } => !!link,
|
||||
)
|
||||
|
||||
return {
|
||||
files: filteredLinks,
|
||||
skippedArchivedCount,
|
||||
}
|
||||
}
|
||||
|
||||
async getStoragePaths(ids: string[]): Promise<GetStoragePathsResponse> {
|
||||
if (ids.length === 0) return { paths: [] }
|
||||
|
||||
const targetAssetIds = await this.resolveLatestVersionIds(ids)
|
||||
const uniqueTargetIds = Array.from(new Set(targetAssetIds))
|
||||
|
||||
const assets = await this.prismaClient.asset.findMany({
|
||||
where: {
|
||||
id: { in: uniqueTargetIds },
|
||||
isDeleted: false,
|
||||
type: {
|
||||
notIn: [AssetType.folder, AssetType.root, AssetType.share_root, AssetType.share],
|
||||
},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
status: true,
|
||||
storageKey: {
|
||||
select: {
|
||||
key: true,
|
||||
isArchived: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const defaultBackend = getStorageBackend()
|
||||
const mainBucket = process.env.S3_BUCKET || 'shumai'
|
||||
const paths: StoragePathItem[] = []
|
||||
|
||||
for (const asset of assets) {
|
||||
const sk = asset.storageKey
|
||||
if (!sk?.key) continue
|
||||
|
||||
const isArchived =
|
||||
asset.status === AssetStatus.archived ||
|
||||
asset.status === AssetStatus.archiving ||
|
||||
asset.status === AssetStatus.restoring ||
|
||||
sk.isArchived
|
||||
|
||||
const archivePathInfo = isArchived ? getArchiveStoragePath(sk.key) : null
|
||||
if (archivePathInfo) {
|
||||
paths.push({
|
||||
id: asset.id,
|
||||
name: asset.name,
|
||||
path: archivePathInfo.path,
|
||||
backend: archivePathInfo.backend,
|
||||
isArchived: true,
|
||||
})
|
||||
} else {
|
||||
const storagePath = s3Service.getStoragePath(mainBucket, sk.key)
|
||||
paths.push({
|
||||
id: asset.id,
|
||||
name: asset.name,
|
||||
path: storagePath,
|
||||
backend: defaultBackend,
|
||||
isArchived,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return { paths }
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2772,7 +3037,13 @@ export class AssetService {
|
||||
const targetAssetId = await this.resolveLatestVersionId(assetId)
|
||||
const asset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: targetAssetId },
|
||||
select: { id: true, name: true, parentId: true, storageKey: { select: { key: true } } },
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
status: true,
|
||||
parentId: true,
|
||||
storageKey: { select: { key: true, isArchived: true } },
|
||||
},
|
||||
})
|
||||
|
||||
if (!asset?.storageKey?.key) {
|
||||
@@ -2786,6 +3057,8 @@ export class AssetService {
|
||||
let storageKey = asset.storageKey.key
|
||||
let assetDir = storageKey.substring(0, storageKey.lastIndexOf('/') + 1)
|
||||
let targetName = asset.name
|
||||
let targetStatus = asset.status
|
||||
let targetIsArchived = asset.storageKey.isArchived
|
||||
|
||||
if (!key.startsWith(assetDir) && asset.parentId) {
|
||||
// Check if requested key belongs to another version in the same version stack
|
||||
@@ -2795,11 +3068,17 @@ export class AssetService {
|
||||
isDeleted: false,
|
||||
storageKey: { key: { startsWith: key.substring(0, key.lastIndexOf('/') + 1) } },
|
||||
},
|
||||
select: { name: true, storageKey: { select: { key: true } } },
|
||||
select: {
|
||||
name: true,
|
||||
status: true,
|
||||
storageKey: { select: { key: true, isArchived: true } },
|
||||
},
|
||||
})
|
||||
if (sibling?.storageKey?.key) {
|
||||
storageKey = sibling.storageKey.key
|
||||
assetDir = storageKey.substring(0, storageKey.lastIndexOf('/') + 1)
|
||||
targetStatus = sibling.status
|
||||
targetIsArchived = sibling.storageKey.isArchived
|
||||
if (sibling.name) {
|
||||
targetName = sibling.name
|
||||
}
|
||||
@@ -2810,6 +3089,19 @@ export class AssetService {
|
||||
throw new Error('Key does not belong to this asset')
|
||||
}
|
||||
|
||||
if (key === storageKey) {
|
||||
if (
|
||||
targetStatus === AssetStatus.archiving ||
|
||||
targetStatus === AssetStatus.archived ||
|
||||
targetStatus === AssetStatus.restoring ||
|
||||
targetIsArchived
|
||||
) {
|
||||
throw new HTTPException(409, {
|
||||
message: 'Original file is archived. Restore the asset to download the original file.',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
let downloadFilename: string | undefined
|
||||
if (key === storageKey) {
|
||||
downloadFilename = targetName ?? undefined
|
||||
@@ -2841,6 +3133,24 @@ export class AssetService {
|
||||
return s3Service.presign(process.env.S3_BUCKET || 'shumai', key, 'GET', true, downloadFilename)
|
||||
}
|
||||
|
||||
async archiveAssets(
|
||||
assetIds: string[],
|
||||
options?: {
|
||||
suppressBusyErrors?: boolean
|
||||
autoArchivePolicy?: {
|
||||
method: 'not_updated' | 'after_created'
|
||||
cutoffDate: Date
|
||||
restoreCutoffDate: Date
|
||||
}
|
||||
},
|
||||
): Promise<ArchiveAssetsResponse> {
|
||||
return assetArchiveService.archiveAssets(assetIds, options)
|
||||
}
|
||||
|
||||
async restoreArchivedAssets(assetIds: string[]): Promise<RestoreArchivedAssetsResponse> {
|
||||
return assetArchiveService.restoreArchivedAssets(assetIds)
|
||||
}
|
||||
|
||||
async toPreviewInfo(
|
||||
asset: Asset | AssetWithIncludes,
|
||||
previewFormat?: PreviewFormat,
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import {
|
||||
AssetStatus,
|
||||
prisma,
|
||||
Prisma,
|
||||
WorkflowTaskType,
|
||||
WorkflowTaskStatus,
|
||||
type User,
|
||||
} from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { assetService } from './asset'
|
||||
import { assetArchiveService } from './asset-archive'
|
||||
import { resetArchiveStorageServiceForTests } from '../s3/archive-storage'
|
||||
import {
|
||||
AutoArchiveJobService,
|
||||
autoArchiveJobService,
|
||||
getAutoArchiveIntervalMs,
|
||||
} from './auto-archive-job'
|
||||
|
||||
describe('AutoArchiveJobService', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
const archiveRoot = path.join(process.cwd(), 'data-test-auto-archive')
|
||||
let originalEnv: NodeJS.ProcessEnv
|
||||
let owner: User
|
||||
let teamId: string
|
||||
let projectId: string
|
||||
let rootFolderId: string
|
||||
|
||||
beforeEach(async () => {
|
||||
originalEnv = { ...process.env }
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
|
||||
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
|
||||
resetArchiveStorageServiceForTests()
|
||||
|
||||
if (fs.existsSync(archiveRoot)) {
|
||||
fs.rmSync(archiveRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
owner = await prisma.user.create({
|
||||
data: { name: 'Auto Archive Owner', email: 'auto-archive-owner@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Auto Archive Team',
|
||||
settings: {
|
||||
autoArchive: {
|
||||
enabled: true,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
teamId = team.id
|
||||
await prisma.teamMember.create({ data: { teamId, userId: owner.id, role: 'owner' } })
|
||||
|
||||
const project = await projectService.createProject(owner, { teamId, name: 'Archive Project' })
|
||||
projectId = project.id
|
||||
rootFolderId = project.rootFolder!
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
resetArchiveStorageServiceForTests()
|
||||
if (fs.existsSync(archiveRoot)) {
|
||||
fs.rmSync(archiveRoot, { recursive: true, force: true })
|
||||
}
|
||||
autoArchiveJobService.stop()
|
||||
})
|
||||
|
||||
async function createTestFile(params: {
|
||||
name: string
|
||||
key: string
|
||||
createdAt?: Date
|
||||
updatedAt?: Date
|
||||
status?: AssetStatus
|
||||
}) {
|
||||
const asset = await assetService.createAsset({
|
||||
name: params.name,
|
||||
type: 'file',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
key: params.key,
|
||||
sizeByte: 2048,
|
||||
contentType: 'video/mp4',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
|
||||
const updated = await prisma.asset.update({
|
||||
where: { id: asset.id },
|
||||
data: {
|
||||
status: params.status ?? AssetStatus.processed,
|
||||
createdAt: params.createdAt ?? new Date(),
|
||||
updatedAt: params.updatedAt ?? new Date(),
|
||||
},
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
return updated
|
||||
}
|
||||
|
||||
describe('getAutoArchiveIntervalMs', () => {
|
||||
it('returns default 1 hour (3600000ms) when env not specified', () => {
|
||||
delete process.env.AUTO_ARCHIVE_INTERVAL_MINUTES
|
||||
expect(getAutoArchiveIntervalMs()).toBe(3600000)
|
||||
})
|
||||
|
||||
it('parses integer minute strings', () => {
|
||||
process.env.AUTO_ARCHIVE_INTERVAL_MINUTES = '30'
|
||||
expect(getAutoArchiveIntervalMs()).toBe(1800000)
|
||||
})
|
||||
|
||||
it('parses decimal floating point minute strings', () => {
|
||||
process.env.AUTO_ARCHIVE_INTERVAL_MINUTES = '0.02'
|
||||
expect(getAutoArchiveIntervalMs()).toBe(1200)
|
||||
})
|
||||
})
|
||||
|
||||
it('skips scan when archive storage is not configured', async () => {
|
||||
delete process.env.ARCHIVE_STORAGE_BACKEND
|
||||
resetArchiveStorageServiceForTests()
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
expect(res.archivedCount).toBe(0)
|
||||
})
|
||||
|
||||
it('skips teams where autoArchive is disabled', async () => {
|
||||
await prisma.team.update({
|
||||
where: { id: teamId },
|
||||
data: {
|
||||
settings: {
|
||||
autoArchive: {
|
||||
enabled: false,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const oldDate = new Date(Date.now() - 40 * 86_400_000)
|
||||
await createTestFile({
|
||||
name: 'old-file.mp4',
|
||||
key: 'files/test/old-file.mp4',
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
expect(res.archivedCount).toBe(0)
|
||||
})
|
||||
|
||||
it('archives assets exceeding not_updated threshold while leaving recent assets untouched', async () => {
|
||||
const oldDate = new Date(Date.now() - 35 * 86_400_000)
|
||||
const recentDate = new Date(Date.now() - 5 * 86_400_000)
|
||||
|
||||
const oldFile = await createTestFile({
|
||||
name: 'old-file.mp4',
|
||||
key: 'files/test/old-file.mp4',
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
|
||||
const recentFile = await createTestFile({
|
||||
name: 'recent-file.mp4',
|
||||
key: 'files/test/recent-file.mp4',
|
||||
updatedAt: recentDate,
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
|
||||
expect(res.archivedCount).toBe(1)
|
||||
|
||||
const refreshedOld = await prisma.asset.findUniqueOrThrow({ where: { id: oldFile.id } })
|
||||
const refreshedRecent = await prisma.asset.findUniqueOrThrow({ where: { id: recentFile.id } })
|
||||
|
||||
expect(refreshedOld.status).toBe(AssetStatus.archiving)
|
||||
expect(refreshedRecent.status).toBe(AssetStatus.processed)
|
||||
|
||||
const task = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: oldFile.id, type: WorkflowTaskType.archive_asset },
|
||||
})
|
||||
expect(task).not.toBeNull()
|
||||
expect(task?.status).toBe(WorkflowTaskStatus.pending)
|
||||
})
|
||||
|
||||
it('archives assets exceeding after_created threshold and respects postRestoreGraceDays', async () => {
|
||||
await prisma.team.update({
|
||||
where: { id: teamId },
|
||||
data: {
|
||||
settings: {
|
||||
autoArchive: {
|
||||
enabled: true,
|
||||
method: 'after_created',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 14,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const oldCreatedDate = new Date(Date.now() - 45 * 86_400_000)
|
||||
const recentlyRestoredDate = new Date(Date.now() - 2 * 86_400_000)
|
||||
const oldRestoredDate = new Date(Date.now() - 20 * 86_400_000)
|
||||
|
||||
// Old creation, but restored 2 days ago (within 14-day grace period) -> should NOT archive
|
||||
const restoredRecently = await createTestFile({
|
||||
name: 'restored-recently.mp4',
|
||||
key: 'files/test/restored-recently.mp4',
|
||||
createdAt: oldCreatedDate,
|
||||
updatedAt: recentlyRestoredDate,
|
||||
})
|
||||
|
||||
// Old creation, restored 20 days ago (> 14-day grace period) -> SHOULD archive
|
||||
const restoredLongAgo = await createTestFile({
|
||||
name: 'restored-long-ago.mp4',
|
||||
key: 'files/test/restored-long-ago.mp4',
|
||||
createdAt: oldCreatedDate,
|
||||
updatedAt: oldRestoredDate,
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
|
||||
expect(res.archivedCount).toBe(1)
|
||||
|
||||
const refreshedRecently = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: restoredRecently.id },
|
||||
})
|
||||
const refreshedLongAgo = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: restoredLongAgo.id },
|
||||
})
|
||||
|
||||
expect(refreshedRecently.status).toBe(AssetStatus.processed)
|
||||
expect(refreshedLongAgo.status).toBe(AssetStatus.archiving)
|
||||
})
|
||||
|
||||
it('skips storage key if any sibling file sharing the key was recently modified', async () => {
|
||||
const oldDate = new Date(Date.now() - 40 * 86_400_000)
|
||||
const recentDate = new Date()
|
||||
|
||||
const sharedKey = 'files/test/shared-video.mp4'
|
||||
|
||||
const primaryFile = await createTestFile({
|
||||
name: 'primary.mp4',
|
||||
key: sharedKey,
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
|
||||
// Create a sibling copy sharing the same storageKeyId but modified recently
|
||||
const siblingCopy = await assetService.createAsset({
|
||||
name: 'sibling-copy.mp4',
|
||||
type: 'file',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
key: sharedKey,
|
||||
sizeByte: 2048,
|
||||
contentType: 'video/mp4',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
|
||||
await prisma.asset.update({
|
||||
where: { id: siblingCopy.id },
|
||||
data: {
|
||||
status: AssetStatus.processed,
|
||||
updatedAt: recentDate,
|
||||
},
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
|
||||
expect(res.archivedCount).toBe(0)
|
||||
|
||||
const refreshedPrimary = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: primaryFile.id },
|
||||
})
|
||||
expect(refreshedPrimary.status).toBe(AssetStatus.processed)
|
||||
})
|
||||
|
||||
it('handles race conditions: concurrent archive attempts cleanly serialize without duplicate tasks', async () => {
|
||||
const oldDate = new Date(Date.now() - 50 * 86_400_000)
|
||||
const file = await createTestFile({
|
||||
name: 'concurrent-test.mp4',
|
||||
key: 'files/test/concurrent-test.mp4',
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
|
||||
// Trigger auto-archive and manual archive concurrently
|
||||
const [jobRes, manualRes] = await Promise.all([
|
||||
job.processAutoArchive(),
|
||||
assetService.archiveAssets([file.id], { suppressBusyErrors: true }),
|
||||
])
|
||||
|
||||
// Together they should archive the asset exactly once
|
||||
const totalArchived = jobRes.archivedCount + manualRes.archivedAssetIds.length
|
||||
expect(totalArchived).toBe(1)
|
||||
|
||||
// Verify exactly ONE WorkflowTask was created
|
||||
const tasks = await prisma.workflowTask.findMany({
|
||||
where: { assetId: file.id, type: WorkflowTaskType.archive_asset },
|
||||
})
|
||||
expect(tasks).toHaveLength(1)
|
||||
|
||||
const refreshedFile = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
|
||||
expect(refreshedFile.status).toBe(AssetStatus.archiving)
|
||||
})
|
||||
|
||||
it('prevents candidate starvation: storage keys with stuck siblings are ignored so eligible candidates are not blocked', async () => {
|
||||
const oldDate = new Date(Date.now() - 40 * 86_400_000)
|
||||
|
||||
// Key 1: has an old file AND a stuck sibling with status 'failed'
|
||||
const stuckKey = 'files/test/stuck-sibling.mp4'
|
||||
const fileWithStuckSibling = await createTestFile({
|
||||
name: 'stuck-primary.mp4',
|
||||
key: stuckKey,
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
const stuckSibling = await assetService.createAsset({
|
||||
name: 'stuck-sibling-failed.mp4',
|
||||
type: 'file',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
key: stuckKey,
|
||||
sizeByte: 2048,
|
||||
contentType: 'video/mp4',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: stuckSibling.id },
|
||||
data: {
|
||||
status: AssetStatus.failed,
|
||||
updatedAt: oldDate,
|
||||
},
|
||||
})
|
||||
|
||||
// Key 2: fully eligible old file
|
||||
const eligibleFile = await createTestFile({
|
||||
name: 'eligible-siblingless.mp4',
|
||||
key: 'files/test/eligible-siblingless.mp4',
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
// With batchSize 1, the query must pick the eligible file instead of getting stuck on Key 1
|
||||
const res = await job.processAutoArchive(1)
|
||||
|
||||
expect(res.archivedCount).toBe(1)
|
||||
|
||||
const refreshedEligible = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: eligibleFile.id },
|
||||
})
|
||||
expect(refreshedEligible.status).toBe(AssetStatus.archiving)
|
||||
|
||||
const refreshedStuck = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: fileWithStuckSibling.id },
|
||||
})
|
||||
expect(refreshedStuck.status).toBe(AssetStatus.processed)
|
||||
})
|
||||
|
||||
it('skips scan when another process holds the transaction advisory lock', async () => {
|
||||
const queryRawSpy = vi.spyOn(prisma, '$queryRaw').mockImplementation((query: unknown) => {
|
||||
const sqlText = String((query as { strings?: string[] })?.strings || '')
|
||||
if (sqlText.includes('pg_try_advisory_xact_lock')) {
|
||||
return Promise.resolve([{ locked: false }]) as unknown as Prisma.PrismaPromise<unknown>
|
||||
}
|
||||
queryRawSpy.mockRestore()
|
||||
return prisma.$queryRaw(query as TemplateStringsArray)
|
||||
})
|
||||
|
||||
try {
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
expect(res.archivedCount).toBe(0)
|
||||
} finally {
|
||||
queryRawSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('logs info level message on job start and finish with totalArchived count', async () => {
|
||||
const infoSpy = vi.spyOn(logger, 'info')
|
||||
|
||||
const oldDate = new Date(Date.now() - 40 * 86_400_000)
|
||||
await createTestFile({
|
||||
name: 'log-test.mp4',
|
||||
key: 'files/test/log-test.mp4',
|
||||
updatedAt: oldDate,
|
||||
})
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
|
||||
expect(res.archivedCount).toBe(1)
|
||||
expect(infoSpy).toHaveBeenCalledWith('Auto-archive job started')
|
||||
expect(infoSpy).toHaveBeenCalledWith({ totalArchived: 1 }, 'Auto-archive job finished')
|
||||
|
||||
infoSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('does not log info level messages when no files are eligible to archive', async () => {
|
||||
const infoSpy = vi.spyOn(logger, 'info')
|
||||
|
||||
const job = new AutoArchiveJobService()
|
||||
const res = await job.processAutoArchive()
|
||||
|
||||
expect(res.archivedCount).toBe(0)
|
||||
expect(infoSpy).not.toHaveBeenCalledWith('Auto-archive job started')
|
||||
expect(infoSpy).not.toHaveBeenCalledWith(expect.anything(), 'Auto-archive job finished')
|
||||
|
||||
infoSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('does not archive non-file assets even if their status is processed and age criteria met', async () => {
|
||||
const oldDate = new Date(Date.now() - 50 * 86_400_000)
|
||||
|
||||
const folder = await assetService.createAsset({
|
||||
name: 'Old Folder',
|
||||
type: 'folder',
|
||||
projectId,
|
||||
parentId: rootFolderId,
|
||||
creatorId: owner.id,
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: folder.id },
|
||||
data: {
|
||||
status: AssetStatus.processed,
|
||||
updatedAt: oldDate,
|
||||
createdAt: oldDate,
|
||||
},
|
||||
})
|
||||
|
||||
const res = await assetArchiveService.archiveAssets([folder.id], {
|
||||
suppressBusyErrors: true,
|
||||
autoArchivePolicy: {
|
||||
method: 'not_updated',
|
||||
cutoffDate: new Date(),
|
||||
restoreCutoffDate: new Date(),
|
||||
},
|
||||
})
|
||||
|
||||
expect(res.archivedAssetIds).toHaveLength(0)
|
||||
|
||||
const refreshedFolder = await prisma.asset.findUniqueOrThrow({
|
||||
where: { id: folder.id },
|
||||
})
|
||||
expect(refreshedFolder.status).toBe(AssetStatus.processed)
|
||||
})
|
||||
|
||||
it('can start and stop without errors', () => {
|
||||
const job = new AutoArchiveJobService()
|
||||
job.start(1000, 5000)
|
||||
job.stop()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,197 @@
|
||||
import { prisma, Prisma } from '@shumai/db'
|
||||
import { isArchiveStorageEnabled } from '@shumai/core/src/s3/archive-storage'
|
||||
import { assetArchiveService } from '@shumai/core/src/asset/asset-archive'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
export function getAutoArchiveIntervalMs(): number {
|
||||
const raw = process.env.AUTO_ARCHIVE_INTERVAL_MINUTES
|
||||
if (raw) {
|
||||
const parsed = Number.parseFloat(raw)
|
||||
if (!Number.isNaN(parsed) && parsed > 0) {
|
||||
return Math.round(parsed * 60 * 1000)
|
||||
}
|
||||
}
|
||||
return 60 * 60 * 1000 // default 1 hour
|
||||
}
|
||||
|
||||
interface TeamArchiveBatch {
|
||||
teamId: string
|
||||
assetIds: string[]
|
||||
policy: {
|
||||
method: 'not_updated' | 'after_created'
|
||||
cutoffDate: Date
|
||||
restoreCutoffDate: Date
|
||||
}
|
||||
}
|
||||
|
||||
export class AutoArchiveJobService {
|
||||
private isRunning = false
|
||||
private isProcessing = false
|
||||
private timer: NodeJS.Timeout | null = null
|
||||
|
||||
start(initialDelayMs = 60_000, intervalMs?: number): void {
|
||||
if (this.isRunning) return
|
||||
this.isRunning = true
|
||||
logger.info('Starting auto-archive background job')
|
||||
|
||||
const run = async () => {
|
||||
if (!this.isRunning) return
|
||||
try {
|
||||
await this.processAutoArchive()
|
||||
} catch (err: unknown) {
|
||||
logger.error({ err }, 'Error in auto-archive background job')
|
||||
}
|
||||
|
||||
if (this.isRunning) {
|
||||
const nextInterval = intervalMs ?? getAutoArchiveIntervalMs()
|
||||
this.timer = setTimeout(run, nextInterval)
|
||||
}
|
||||
}
|
||||
|
||||
this.timer = setTimeout(run, initialDelayMs)
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.isRunning = false
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
logger.info('Stopped auto-archive background job')
|
||||
}
|
||||
|
||||
async processAutoArchive(batchSize = 100): Promise<{ archivedCount: number }> {
|
||||
if (!isArchiveStorageEnabled()) {
|
||||
logger.debug('Archive storage is not configured; skipping auto-archive run')
|
||||
return { archivedCount: 0 }
|
||||
}
|
||||
|
||||
if (this.isProcessing) {
|
||||
logger.debug('Auto-archive job turn already in progress, skipping')
|
||||
return { archivedCount: 0 }
|
||||
}
|
||||
|
||||
this.isProcessing = true
|
||||
|
||||
try {
|
||||
let totalArchived = 0
|
||||
|
||||
// Phase 1: Query & Scan under transaction-scoped advisory lock.
|
||||
// Automatically released as soon as this transaction completes.
|
||||
const teamBatches = await prisma.$transaction(async (tx) => {
|
||||
const lockRes = await tx.$queryRaw<Array<{ locked: boolean }>>`
|
||||
SELECT pg_try_advisory_xact_lock(hashtext('auto_archive_job')) AS locked
|
||||
`
|
||||
if (!lockRes || !lockRes[0]?.locked) {
|
||||
logger.debug(
|
||||
'Could not acquire transaction advisory lock for auto-archive scan, skipping',
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
// Query only teams where autoArchive is enabled via JSON filter
|
||||
const eligibleTeams = await tx.team.findMany({
|
||||
where: {
|
||||
settings: {
|
||||
path: ['autoArchive', 'enabled'],
|
||||
equals: true,
|
||||
},
|
||||
},
|
||||
select: { id: true, settings: true },
|
||||
})
|
||||
|
||||
if (eligibleTeams.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
const batches: TeamArchiveBatch[] = []
|
||||
|
||||
for (const team of eligibleTeams) {
|
||||
const settings = (team.settings || {}) as PrismaJson.Settings
|
||||
const autoArchive = settings.autoArchive
|
||||
const method = autoArchive?.method || 'not_updated'
|
||||
const days = Math.max(1, autoArchive?.days || 30)
|
||||
const postRestoreGraceDays = Math.max(1, autoArchive?.postRestoreGraceDays || 30)
|
||||
|
||||
const now = Date.now()
|
||||
const cutoffDate = new Date(now - days * 86_400_000)
|
||||
const restoreCutoffDate = new Date(now - postRestoreGraceDays * 86_400_000)
|
||||
|
||||
// Query candidate assets whose storage keys have NO ineligible sibling.
|
||||
// NOT EXISTS eliminates starvation from stuck/recent siblings at the database level.
|
||||
const candidateRows = await tx.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT a.id
|
||||
FROM assets a
|
||||
JOIN projects p ON a.project_id = p.id
|
||||
JOIN storage_keys sk ON a.storage_key_id = sk.id
|
||||
WHERE p.team_id = ${team.id}
|
||||
AND a.type = 'file'::"AssetType"
|
||||
AND a.status = 'processed'::"AssetStatus"
|
||||
AND a.is_deleted = false
|
||||
AND sk.is_archived = false
|
||||
AND ${
|
||||
method === 'not_updated'
|
||||
? Prisma.sql`a.updated_at <= ${cutoffDate}`
|
||||
: Prisma.sql`a.created_at <= ${cutoffDate} AND a.updated_at <= ${restoreCutoffDate}`
|
||||
}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM assets sib
|
||||
WHERE sib.storage_key_id = a.storage_key_id
|
||||
AND sib.is_deleted = false
|
||||
AND (
|
||||
sib.type != 'file'::"AssetType"
|
||||
OR sib.status != 'processed'::"AssetStatus"
|
||||
OR ${
|
||||
method === 'not_updated'
|
||||
? Prisma.sql`sib.updated_at > ${cutoffDate}`
|
||||
: Prisma.sql`sib.created_at > ${cutoffDate} OR sib.updated_at > ${restoreCutoffDate}`
|
||||
}
|
||||
)
|
||||
)
|
||||
ORDER BY a.id ASC
|
||||
LIMIT ${batchSize};
|
||||
`
|
||||
|
||||
if (candidateRows.length > 0) {
|
||||
batches.push({
|
||||
teamId: team.id,
|
||||
assetIds: candidateRows.map((r) => r.id),
|
||||
policy: { method, cutoffDate, restoreCutoffDate },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return batches
|
||||
})
|
||||
|
||||
if (!teamBatches || teamBatches.length === 0) {
|
||||
return { archivedCount: 0 }
|
||||
}
|
||||
|
||||
logger.info('Auto-archive job started')
|
||||
|
||||
// Phase 2: Execute archival in chunks per team batch.
|
||||
// Each asset/storageKey transition is protected by row-level lock and re-verifies policy.
|
||||
const chunkSize = 50
|
||||
|
||||
for (const batch of teamBatches) {
|
||||
for (let i = 0; i < batch.assetIds.length; i += chunkSize) {
|
||||
const chunk = batch.assetIds.slice(i, i + chunkSize)
|
||||
const result = await assetArchiveService.archiveAssets(chunk, {
|
||||
suppressBusyErrors: true,
|
||||
autoArchivePolicy: batch.policy,
|
||||
})
|
||||
totalArchived += result.archivedAssetIds.length
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ totalArchived }, 'Auto-archive job finished')
|
||||
return { archivedCount: totalArchived }
|
||||
} finally {
|
||||
this.isProcessing = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const autoArchiveJobService = new AutoArchiveJobService()
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { auth, authService } from './auth'
|
||||
import { auth, authService, isLoopbackHost, isMcpResourceValid } from './auth'
|
||||
import { userMetadataService } from '../user-metadata/user-metadata'
|
||||
import { teamService } from '@shumai/core/src/team/team'
|
||||
|
||||
@@ -326,3 +326,48 @@ describe('AuthService Password Reset', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('MCP Resource Validation & Loopback Checking', () => {
|
||||
it('correctly identifies loopback hostnames', () => {
|
||||
expect(isLoopbackHost('localhost')).toBe(true)
|
||||
expect(isLoopbackHost('127.0.0.1')).toBe(true)
|
||||
expect(isLoopbackHost('127.0.0.254')).toBe(true)
|
||||
expect(isLoopbackHost('[::1]')).toBe(true)
|
||||
|
||||
expect(isLoopbackHost('43.133.100.178')).toBe(false)
|
||||
expect(isLoopbackHost('192.168.1.100')).toBe(false)
|
||||
expect(isLoopbackHost('10.0.0.1')).toBe(false)
|
||||
expect(isLoopbackHost('example.com')).toBe(false)
|
||||
expect(isLoopbackHost('128.0.0.1')).toBe(false)
|
||||
})
|
||||
|
||||
it('validates MCP resource URLs according to RFC 9728 and MCP security rules', () => {
|
||||
// Loopback HTTP URLs are allowed for development
|
||||
expect(isMcpResourceValid('http://localhost:3000/mcp')).toEqual({ valid: true })
|
||||
expect(isMcpResourceValid('http://127.0.0.1:3000/mcp')).toEqual({ valid: true })
|
||||
expect(isMcpResourceValid('http://[::1]:3000/mcp')).toEqual({ valid: true })
|
||||
|
||||
// HTTPS is allowed for any host (including domains and remote IPs)
|
||||
expect(isMcpResourceValid('https://shumai.example.com/mcp')).toEqual({ valid: true })
|
||||
expect(isMcpResourceValid('https://43.133.100.178:3000/mcp')).toEqual({ valid: true })
|
||||
|
||||
// Remote HTTP IPs and hostnames are rejected (the user issue)
|
||||
const remoteIpResult = isMcpResourceValid('http://43.133.100.178:3000/mcp')
|
||||
expect(remoteIpResult.valid).toBe(false)
|
||||
expect(remoteIpResult.error).toContain(
|
||||
'MCP resource URL must use HTTPS, except for localhost or loopback IP development URLs',
|
||||
)
|
||||
|
||||
const remoteDomainResult = isMcpResourceValid('http://shumai.example.com/mcp')
|
||||
expect(remoteDomainResult.valid).toBe(false)
|
||||
expect(remoteDomainResult.error).toContain(
|
||||
'MCP resource URL must use HTTPS, except for localhost or loopback IP development URLs',
|
||||
)
|
||||
|
||||
// Invalid URL formats
|
||||
expect(isMcpResourceValid('not-a-url').valid).toBe(false)
|
||||
expect(isMcpResourceValid('http://user:pass@localhost:3000/mcp').valid).toBe(false)
|
||||
expect(isMcpResourceValid('http://localhost:3000/mcp#frag').valid).toBe(false)
|
||||
expect(isMcpResourceValid('http://localhost:3000/mcp?query=1').valid).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -8,6 +8,7 @@ import { teamService } from '@shumai/core/src/team/team'
|
||||
import { inviteService } from '@shumai/core/src/invite/invite'
|
||||
import { createAuthMiddleware } from 'better-auth/api'
|
||||
import { userMetadataService } from '@shumai/core/src/user-metadata/user-metadata'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
interface GeneratedResetLink {
|
||||
url: string
|
||||
@@ -22,22 +23,80 @@ export const authBaseUrl = (
|
||||
export const mcpResource = `${authBaseUrl}/mcp`
|
||||
export { hashPassword } from 'better-auth/crypto'
|
||||
|
||||
/**
|
||||
* Checks whether a host is a loopback address (localhost, 127.0.0.0/8, or [::1]).
|
||||
*/
|
||||
export function isLoopbackHost(hostname: string): boolean {
|
||||
const ipv4Octets = hostname.split('.')
|
||||
const isIpv4Loopback =
|
||||
ipv4Octets.length === 4 &&
|
||||
ipv4Octets[0] === '127' &&
|
||||
ipv4Octets.every((octet) => /^\d+$/.test(octet) && Number(octet) <= 255)
|
||||
return hostname === 'localhost' || hostname === '[::1]' || isIpv4Loopback
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates whether an MCP resource URL meets RFC 9728 and MCP security requirements
|
||||
* (HTTPS required unless using a localhost or loopback IP address).
|
||||
*/
|
||||
export function isMcpResourceValid(resource: string): { valid: boolean; error?: string } {
|
||||
if (typeof resource !== 'string' || !resource) {
|
||||
return { valid: false, error: 'MCP resource must be a non-empty string URL' }
|
||||
}
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(resource)
|
||||
} catch {
|
||||
return { valid: false, error: 'MCP resource must be an absolute URL' }
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
return { valid: false, error: 'MCP resource URL must not contain credentials' }
|
||||
}
|
||||
if (resource.includes('#')) {
|
||||
return { valid: false, error: 'MCP resource URL must not contain a fragment' }
|
||||
}
|
||||
if (resource.includes('?')) {
|
||||
return { valid: false, error: 'MCP resource URL must not contain a query' }
|
||||
}
|
||||
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && isLoopbackHost(url.hostname))) {
|
||||
return {
|
||||
valid: false,
|
||||
error:
|
||||
'MCP resource URL must use HTTPS, except for localhost or loopback IP development URLs',
|
||||
}
|
||||
}
|
||||
return { valid: true }
|
||||
}
|
||||
|
||||
const mcpValidation = isMcpResourceValid(mcpResource)
|
||||
export const isMcpOauthEnabled = mcpValidation.valid
|
||||
if (!isMcpOauthEnabled) {
|
||||
logger.error(
|
||||
{ mcpResource, error: mcpValidation.error },
|
||||
'MCP OAuth 2.1 is disabled: resource URL must use HTTPS for remote hosts. The main server and Web UI will continue running normally, but MCP OAuth clients cannot connect until HTTPS is configured.',
|
||||
)
|
||||
}
|
||||
|
||||
export const auth = betterAuth({
|
||||
database: prismaAdapter(prisma, {
|
||||
provider: 'postgresql',
|
||||
}),
|
||||
plugins: [
|
||||
jwt(),
|
||||
mcp({
|
||||
loginPage: '/login',
|
||||
consentPage: '/consent',
|
||||
resource: mcpResource,
|
||||
resources: [{ identifier: mcpResource, allowedScopes: ['mcp'] }],
|
||||
scopes: ['openid', 'profile', 'email', 'offline_access', 'mcp'],
|
||||
allowDynamicClientRegistration: true,
|
||||
allowUnauthenticatedClientRegistration: true,
|
||||
clientRegistrationDefaultApplicationType: 'infer',
|
||||
}),
|
||||
...(isMcpOauthEnabled
|
||||
? [
|
||||
mcp({
|
||||
loginPage: '/login',
|
||||
consentPage: '/consent',
|
||||
resource: mcpResource,
|
||||
resources: [{ identifier: mcpResource, allowedScopes: ['mcp'] }],
|
||||
scopes: ['openid', 'profile', 'email', 'offline_access', 'mcp'],
|
||||
allowDynamicClientRegistration: true,
|
||||
allowUnauthenticatedClientRegistration: true,
|
||||
clientRegistrationDefaultApplicationType: 'infer',
|
||||
}),
|
||||
]
|
||||
: []),
|
||||
],
|
||||
emailAndPassword: {
|
||||
enabled: true,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createGuestToken, verifyGuestToken } from './guest-token'
|
||||
|
||||
describe('guest-token', () => {
|
||||
it('creates and verifies a valid token for the expected shareId', () => {
|
||||
const token = createGuestToken({ userId: 'u1', shareId: 'share1' })
|
||||
const verified = verifyGuestToken(token, 'share1')
|
||||
expect(verified).not.toBeNull()
|
||||
expect(verified?.userId).toBe('u1')
|
||||
expect(verified?.shareId).toBe('share1')
|
||||
})
|
||||
|
||||
it('rejects a token when verified against a different shareId', () => {
|
||||
const token = createGuestToken({ userId: 'u1', shareId: 'share1' })
|
||||
const verified = verifyGuestToken(token, 'share2')
|
||||
expect(verified).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects an expired token', () => {
|
||||
const token = createGuestToken({ userId: 'u1', shareId: 'share1' }, -10)
|
||||
const verified = verifyGuestToken(token, 'share1')
|
||||
expect(verified).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects a tampered signature', () => {
|
||||
const token = createGuestToken({ userId: 'u1', shareId: 'share1' })
|
||||
const [payload] = token.split('.')
|
||||
const tampered = `${payload}.invalidsignature`
|
||||
const verified = verifyGuestToken(tampered, 'share1')
|
||||
expect(verified).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects malformed token strings', () => {
|
||||
expect(verifyGuestToken('', 'share1')).toBeNull()
|
||||
expect(verifyGuestToken('singlepart', 'share1')).toBeNull()
|
||||
expect(verifyGuestToken('a.b.c', 'share1')).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import * as crypto from 'crypto'
|
||||
|
||||
export interface GuestTokenPayload {
|
||||
userId: string
|
||||
shareId: string
|
||||
expiresAt: number
|
||||
}
|
||||
|
||||
const GUEST_TOKEN_CONTEXT = 'shumai-guest-token:v1'
|
||||
const DEFAULT_TTL_SECONDS = 30 * 24 * 60 * 60 // 30 days
|
||||
|
||||
function getSecret(): string {
|
||||
return process.env.BETTER_AUTH_SECRET || 'shumai-guest-secret'
|
||||
}
|
||||
|
||||
function signPayload(payloadB64: string): string {
|
||||
return crypto
|
||||
.createHmac('sha256', getSecret())
|
||||
.update(`${GUEST_TOKEN_CONTEXT}:${payloadB64}`)
|
||||
.digest('base64url')
|
||||
}
|
||||
|
||||
export function createGuestToken(
|
||||
params: { userId: string; shareId: string },
|
||||
ttlSeconds: number = DEFAULT_TTL_SECONDS,
|
||||
): string {
|
||||
const payload: GuestTokenPayload = {
|
||||
userId: params.userId,
|
||||
shareId: params.shareId,
|
||||
expiresAt: Date.now() + ttlSeconds * 1000,
|
||||
}
|
||||
const payloadB64 = Buffer.from(JSON.stringify(payload)).toString('base64url')
|
||||
const signature = signPayload(payloadB64)
|
||||
return `${payloadB64}.${signature}`
|
||||
}
|
||||
|
||||
export function verifyGuestToken(token: string, expectedShareId: string): GuestTokenPayload | null {
|
||||
if (!token) return null
|
||||
const [payloadB64, signature, ...rest] = token.split('.')
|
||||
if (!payloadB64 || !signature || rest.length > 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
const expectedSignature = signPayload(payloadB64)
|
||||
const actualBuf = Buffer.from(signature)
|
||||
const expectedBuf = Buffer.from(expectedSignature)
|
||||
|
||||
if (actualBuf.length !== expectedBuf.length || !crypto.timingSafeEqual(actualBuf, expectedBuf)) {
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(payloadB64, 'base64url').toString('utf8'),
|
||||
) as GuestTokenPayload
|
||||
if (!payload.userId || !payload.shareId || typeof payload.expiresAt !== 'number') {
|
||||
return null
|
||||
}
|
||||
if (payload.shareId !== expectedShareId) {
|
||||
return null
|
||||
}
|
||||
if (Date.now() > payload.expiresAt) {
|
||||
return null
|
||||
}
|
||||
return payload
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
export * from './agent/agent'
|
||||
export * from './asset/asset'
|
||||
export * from './asset/asset-archive'
|
||||
export * from './asset/auto-archive-job'
|
||||
export * from './auth/auth'
|
||||
export * from './authz/authz'
|
||||
export * from './collection/collection'
|
||||
|
||||
@@ -141,4 +141,33 @@ describe('Invite Service', () => {
|
||||
expect(updatedTm?.scope).toBe('team')
|
||||
expect(updatedTm?.role).toBe('editor')
|
||||
})
|
||||
|
||||
it('atomically claims invite and rejects concurrent redemptions', async () => {
|
||||
const inv = await inviteService.createTeamInvite({
|
||||
teamId: team.id,
|
||||
role: 'editor',
|
||||
inviterId: admin.id,
|
||||
})
|
||||
|
||||
const userA = await prisma.user.create({
|
||||
data: { name: 'userA', email: 'userA-invite@example.com', password: 'pass' },
|
||||
})
|
||||
const userB = await prisma.user.create({
|
||||
data: { name: 'userB', email: 'userB-invite@example.com', password: 'pass' },
|
||||
})
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
inviteService.consumeInvite(inv.code, userA.id),
|
||||
inviteService.consumeInvite(inv.code, userB.id),
|
||||
])
|
||||
|
||||
const fulfilled = results.filter((r) => r.status === 'fulfilled')
|
||||
const rejected = results.filter((r) => r.status === 'rejected')
|
||||
|
||||
expect(fulfilled.length).toBe(1)
|
||||
expect(rejected.length).toBe(1)
|
||||
if (rejected[0].status === 'rejected') {
|
||||
expect(rejected[0].reason.message).toContain('invite code already used')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -84,7 +84,11 @@ export class InviteService {
|
||||
include: { team: true, project: true },
|
||||
})
|
||||
|
||||
if (inv.used) {
|
||||
const claimed = await tx.invite.updateMany({
|
||||
where: { id: inv.id, used: false },
|
||||
data: { used: true },
|
||||
})
|
||||
if (claimed.count !== 1) {
|
||||
throw new Error('invite code already used')
|
||||
}
|
||||
|
||||
@@ -155,11 +159,6 @@ export class InviteService {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
await tx.invite.update({
|
||||
where: { id: inv.id },
|
||||
data: { used: true },
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { AssetStatus, prisma, type User } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { projectService } from '@shumai/core/src/project/project'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { resetArchiveStorageServiceForTests } from '@shumai/core/src/s3/archive-storage'
|
||||
import {
|
||||
archiveAssetsOp,
|
||||
restoreArchivedAssetsOp,
|
||||
runOperation,
|
||||
type OperationContext,
|
||||
} from './index'
|
||||
|
||||
describe('archive_assets and restore_archived_assets Operations', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
const archiveRoot = path.join(process.cwd(), 'data-test-ops-archive')
|
||||
let originalEnv: NodeJS.ProcessEnv
|
||||
let owner: User
|
||||
let editor: User
|
||||
let reviewer: User
|
||||
let outsider: User
|
||||
let teamId: string
|
||||
let projectId: string
|
||||
let rootFolderId: string
|
||||
|
||||
const as = (user: User, via: OperationContext['via'] = 'cli'): OperationContext => ({
|
||||
userId: user.id,
|
||||
via,
|
||||
})
|
||||
|
||||
beforeEach(async () => {
|
||||
originalEnv = { ...process.env }
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
|
||||
process.env.ARCHIVE_LOCAL_ROOT = archiveRoot
|
||||
resetArchiveStorageServiceForTests()
|
||||
|
||||
owner = await prisma.user.create({
|
||||
data: { name: 'Owner', email: 'arch-ops-owner@test.com', password: 'pw' },
|
||||
})
|
||||
editor = await prisma.user.create({
|
||||
data: { name: 'Editor', email: 'arch-ops-editor@test.com', password: 'pw' },
|
||||
})
|
||||
reviewer = await prisma.user.create({
|
||||
data: { name: 'Reviewer', email: 'arch-ops-reviewer@test.com', password: 'pw' },
|
||||
})
|
||||
outsider = await prisma.user.create({
|
||||
data: { name: 'Outsider', email: 'arch-ops-outsider@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 'Arch Ops Team' } })
|
||||
teamId = team.id
|
||||
await prisma.teamMember.create({ data: { teamId, userId: owner.id, role: 'owner' } })
|
||||
await prisma.teamMember.create({ data: { teamId, userId: editor.id, role: 'editor' } })
|
||||
await prisma.teamMember.create({ data: { teamId, userId: reviewer.id, role: 'reviewer' } })
|
||||
|
||||
const project = await projectService.createProject(owner, { teamId, name: 'Arch Ops Project' })
|
||||
projectId = project.id
|
||||
rootFolderId = project.rootFolder!
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
resetArchiveStorageServiceForTests()
|
||||
if (fs.existsSync(archiveRoot)) {
|
||||
fs.rmSync(archiveRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('allows editor and owner to archive and restore assets and records audit entries', async () => {
|
||||
const file = await assetService.createAsset({
|
||||
name: 'take1.mov',
|
||||
type: 'file',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
key: 'files/01OP/take1.mov',
|
||||
sizeByte: 4096,
|
||||
contentType: 'video/quicktime',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: file.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
})
|
||||
|
||||
const archiveOut = await runOperation(archiveAssetsOp, as(editor, 'api'), {
|
||||
assetIds: [file.id],
|
||||
})
|
||||
expect(archiveOut.archivedAssetIds).toEqual([file.id])
|
||||
expect(archiveOut.taskIds).toHaveLength(1)
|
||||
expect(archiveOut.skippedCount).toBe(0)
|
||||
|
||||
const archiveAudit = await prisma.auditLog.findMany({
|
||||
where: { action: AuditAction.asset_archive },
|
||||
})
|
||||
expect(archiveAudit).toHaveLength(1)
|
||||
expect(archiveAudit[0]).toMatchObject({
|
||||
teamId,
|
||||
projectId,
|
||||
userId: editor.id,
|
||||
itemId: file.id,
|
||||
via: 'api',
|
||||
})
|
||||
|
||||
// Transition to archived state to test restore_archived_assets
|
||||
await prisma.storageKey.update({
|
||||
where: { key: 'files/01OP/take1.mov' },
|
||||
data: { isArchived: true },
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: file.id },
|
||||
data: { status: AssetStatus.archived },
|
||||
})
|
||||
|
||||
const restoreOut = await runOperation(restoreArchivedAssetsOp, as(owner, 'mcp'), {
|
||||
assetIds: [file.id],
|
||||
})
|
||||
expect(restoreOut.restoredAssetIds).toEqual([file.id])
|
||||
expect(restoreOut.taskIds).toHaveLength(1)
|
||||
|
||||
const restoreAudit = await prisma.auditLog.findMany({
|
||||
where: { action: AuditAction.asset_restore_archive },
|
||||
})
|
||||
expect(restoreAudit).toHaveLength(1)
|
||||
expect(restoreAudit[0]).toMatchObject({
|
||||
teamId,
|
||||
projectId,
|
||||
userId: owner.id,
|
||||
itemId: file.id,
|
||||
via: 'mcp',
|
||||
})
|
||||
})
|
||||
|
||||
it('denies reviewer and outsider with 403 and causes no side effects or audit entries', async () => {
|
||||
const file = await assetService.createAsset({
|
||||
name: 'protected.mov',
|
||||
type: 'file',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
key: 'files/01PROT/protected.mov',
|
||||
sizeByte: 2048,
|
||||
contentType: 'video/quicktime',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: file.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
})
|
||||
|
||||
for (const user of [reviewer, outsider]) {
|
||||
await expect(
|
||||
runOperation(archiveAssetsOp, as(user), { assetIds: [file.id] }),
|
||||
).rejects.toMatchObject({ status: 403 })
|
||||
}
|
||||
|
||||
const row = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
|
||||
expect(row.status).toBe(AssetStatus.processed)
|
||||
expect(await prisma.auditLog.count({ where: { action: AuditAction.asset_archive } })).toBe(0)
|
||||
|
||||
await prisma.storageKey.update({
|
||||
where: { key: 'files/01PROT/protected.mov' },
|
||||
data: { isArchived: true },
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: file.id },
|
||||
data: { status: AssetStatus.archived },
|
||||
})
|
||||
|
||||
for (const user of [reviewer, outsider]) {
|
||||
await expect(
|
||||
runOperation(restoreArchivedAssetsOp, as(user), { assetIds: [file.id] }),
|
||||
).rejects.toMatchObject({ status: 403 })
|
||||
}
|
||||
|
||||
const archivedRow = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
|
||||
expect(archivedRow.status).toBe(AssetStatus.archived)
|
||||
expect(
|
||||
await prisma.auditLog.count({ where: { action: AuditAction.asset_restore_archive } }),
|
||||
).toBe(0)
|
||||
})
|
||||
|
||||
it('rejects symlinks directly with 400 and causes no side effects', async () => {
|
||||
const file = await assetService.createAsset({
|
||||
name: 'real.mov',
|
||||
type: 'file',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
key: 'files/01REAL/real.mov',
|
||||
sizeByte: 2048,
|
||||
contentType: 'video/quicktime',
|
||||
creatorId: owner.id,
|
||||
})
|
||||
await prisma.asset.update({
|
||||
where: { id: file.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
})
|
||||
|
||||
const symlink = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'symlink.mov',
|
||||
type: 'symlink',
|
||||
parentId: rootFolderId,
|
||||
projectId,
|
||||
targetId: file.id,
|
||||
status: AssetStatus.processed,
|
||||
creatorId: owner.id,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(
|
||||
runOperation(archiveAssetsOp, as(owner), { assetIds: [symlink.id] }),
|
||||
).rejects.toMatchObject({ status: 400, message: 'Symlinks cannot be archived or restored' })
|
||||
|
||||
const unchangedFile = await prisma.asset.findUniqueOrThrow({ where: { id: file.id } })
|
||||
expect(unchangedFile.status).toBe(AssetStatus.processed)
|
||||
expect(await prisma.auditLog.count({ where: { action: AuditAction.asset_archive } })).toBe(0)
|
||||
|
||||
await expect(
|
||||
runOperation(restoreArchivedAssetsOp, as(owner), { assetIds: [symlink.id] }),
|
||||
).rejects.toMatchObject({ status: 400, message: 'Symlinks cannot be archived or restored' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
import { archiveAssets } from '@shumai/operations'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { implementOperation } from '../implement'
|
||||
|
||||
export const archiveAssetsOp = implementOperation(archiveAssets, {
|
||||
authz: (input) =>
|
||||
input.assetIds.map((id) => ({
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Asset,
|
||||
id,
|
||||
})),
|
||||
execute: async (input) => {
|
||||
return assetService.archiveAssets(input.assetIds)
|
||||
},
|
||||
audit: (input) =>
|
||||
input.assetIds.map((assetId) => ({
|
||||
action: AuditAction.asset_archive,
|
||||
itemId: assetId,
|
||||
contextAssetId: assetId,
|
||||
})),
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
import { restoreArchivedAssets } from '@shumai/operations'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { implementOperation } from '../implement'
|
||||
|
||||
export const restoreArchivedAssetsOp = implementOperation(restoreArchivedAssets, {
|
||||
authz: (input) =>
|
||||
input.assetIds.map((id) => ({
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Asset,
|
||||
id,
|
||||
})),
|
||||
execute: async (input) => {
|
||||
return assetService.restoreArchivedAssets(input.assetIds)
|
||||
},
|
||||
audit: (input) =>
|
||||
input.assetIds.map((assetId) => ({
|
||||
action: AuditAction.asset_restore_archive,
|
||||
itemId: assetId,
|
||||
contextAssetId: assetId,
|
||||
})),
|
||||
})
|
||||
@@ -10,6 +10,7 @@ export {
|
||||
} from './implement'
|
||||
export { runOperation } from './run'
|
||||
export { getOperation, getOperationsForSurface, operations } from './registry'
|
||||
export { archiveAssetsOp } from './asset/archive-assets'
|
||||
export { copyAssetsOp } from './asset/copy-assets'
|
||||
export { createFolderOp } from './asset/create-folder'
|
||||
export { deleteAssetOp } from './asset/delete-asset'
|
||||
@@ -24,6 +25,7 @@ export {
|
||||
waitBudgetMs,
|
||||
} from './asset/read-asset'
|
||||
export { renameAssetOp } from './asset/rename-asset'
|
||||
export { restoreArchivedAssetsOp } from './asset/restore-archived-assets'
|
||||
export { searchAssetsOp } from './asset/search-assets'
|
||||
export { createCommentOp } from './comment/create-comment'
|
||||
export { listCommentsOp } from './comment/list-comments'
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { OperationName, Surface } from '@shumai/operations'
|
||||
import { archiveAssetsOp } from './asset/archive-assets'
|
||||
import { copyAssetsOp } from './asset/copy-assets'
|
||||
import { createFolderOp } from './asset/create-folder'
|
||||
import { deleteAssetOp } from './asset/delete-asset'
|
||||
@@ -8,6 +9,7 @@ import { listAssetsOp } from './asset/list-assets'
|
||||
import { moveAssetsOp } from './asset/move-assets'
|
||||
import { getAssetRenderOp, readAssetOp } from './asset/read-asset'
|
||||
import { renameAssetOp } from './asset/rename-asset'
|
||||
import { restoreArchivedAssetsOp } from './asset/restore-archived-assets'
|
||||
import { searchAssetsOp } from './asset/search-assets'
|
||||
import { createCommentOp } from './comment/create-comment'
|
||||
import { listCommentsOp } from './comment/list-comments'
|
||||
@@ -49,6 +51,8 @@ export const operations = defineRegistry([
|
||||
moveAssetsOp,
|
||||
copyAssetsOp,
|
||||
deleteAssetOp,
|
||||
archiveAssetsOp,
|
||||
restoreArchivedAssetsOp,
|
||||
readAssetOp,
|
||||
getAssetRenderOp,
|
||||
downloadAssetOp,
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import {
|
||||
PutObjectCommand,
|
||||
HeadObjectCommand,
|
||||
CopyObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCommand,
|
||||
UploadPartCopyCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
AbortMultipartUploadCommand,
|
||||
RestoreObjectCommand,
|
||||
} from '@aws-sdk/client-s3'
|
||||
import {
|
||||
getArchiveStorageConfig,
|
||||
isArchiveStorageEnabled,
|
||||
isColdStorageRestoreError,
|
||||
LocalArchiveStorageService,
|
||||
parseRestoreHeaderStatus,
|
||||
resetArchiveStorageServiceForTests,
|
||||
S3ArchiveStorageService,
|
||||
} from './archive-storage'
|
||||
import { uploadStreamToS3Client } from './s3'
|
||||
|
||||
const s3ClientConstructorSpy = vi.fn()
|
||||
const s3SendSpy = vi.fn()
|
||||
|
||||
vi.mock('@aws-sdk/client-s3', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@aws-sdk/client-s3')>()
|
||||
return {
|
||||
...actual,
|
||||
S3Client: class {
|
||||
constructor(params: unknown) {
|
||||
s3ClientConstructorSpy(params)
|
||||
}
|
||||
send = s3SendSpy
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
function makeWebStream(chunks: Array<string | Uint8Array>): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) {
|
||||
controller.enqueue(typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk)
|
||||
}
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async function readAllBytes(stream: ReadableStream<Uint8Array>): Promise<Buffer> {
|
||||
const reader = stream.getReader()
|
||||
const parts: Uint8Array[] = []
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
if (value) parts.push(value)
|
||||
}
|
||||
return Buffer.concat(parts)
|
||||
}
|
||||
|
||||
describe('Archive Storage', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env }
|
||||
vi.clearAllMocks()
|
||||
resetArchiveStorageServiceForTests()
|
||||
delete process.env.ARCHIVE_STORAGE_BACKEND
|
||||
delete process.env.ARCHIVE_LOCAL_ROOT
|
||||
delete process.env.ARCHIVE_S3_BUCKET
|
||||
delete process.env.ARCHIVE_AWS_ENDPOINT_URL_S3
|
||||
delete process.env.ARCHIVE_S3_REGION
|
||||
delete process.env.ARCHIVE_S3_ACCESS_KEY_ID
|
||||
delete process.env.ARCHIVE_S3_SECRET_ACCESS_KEY
|
||||
delete process.env.ARCHIVE_S3_FORCE_PATH_STYLE
|
||||
delete process.env.ARCHIVE_S3_STORAGE_CLASS
|
||||
delete process.env.ARCHIVE_S3_USE_RESTORE_OBJECT
|
||||
delete process.env.ARCHIVE_S3_RESTORE_DAYS
|
||||
delete process.env.ARCHIVE_S3_RESTORE_TIER
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
resetArchiveStorageServiceForTests()
|
||||
})
|
||||
|
||||
describe('Environment configuration & validation', () => {
|
||||
it('is disabled when ARCHIVE_STORAGE_BACKEND is unset or empty', () => {
|
||||
expect(isArchiveStorageEnabled()).toBe(false)
|
||||
expect(getArchiveStorageConfig()).toEqual({ enabled: false })
|
||||
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = ' '
|
||||
expect(isArchiveStorageEnabled()).toBe(false)
|
||||
expect(getArchiveStorageConfig()).toEqual({ enabled: false })
|
||||
})
|
||||
|
||||
it('rejects unsupported ARCHIVE_STORAGE_BACKEND values', () => {
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'ftp'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/Invalid ARCHIVE_STORAGE_BACKEND/)
|
||||
})
|
||||
|
||||
it('requires ARCHIVE_LOCAL_ROOT for local archive backend and rejects overlap with main local storage', () => {
|
||||
process.env.STORAGE_BACKEND = 'local'
|
||||
process.env.S3_BUCKET = 'shumai'
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
|
||||
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_LOCAL_ROOT is required/)
|
||||
|
||||
process.env.ARCHIVE_LOCAL_ROOT = 'data/shumai'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/must not overlap/)
|
||||
|
||||
process.env.ARCHIVE_LOCAL_ROOT = 'data/shumai/subdir'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/must not overlap/)
|
||||
|
||||
process.env.ARCHIVE_LOCAL_ROOT = 'data'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/must not overlap/)
|
||||
|
||||
process.env.ARCHIVE_LOCAL_ROOT = 'data-archive'
|
||||
const cfg = getArchiveStorageConfig()
|
||||
expect(cfg).toEqual({
|
||||
enabled: true,
|
||||
backend: 'local',
|
||||
localRoot: path.resolve('data-archive'),
|
||||
isSameS3Target: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('never falls back to main S3_* environment variables when ARCHIVE_STORAGE_BACKEND is s3', () => {
|
||||
process.env.STORAGE_BACKEND = 's3'
|
||||
process.env.S3_BUCKET = 'main-bucket'
|
||||
process.env.S3_ACCESS_KEY_ID = 'main-key'
|
||||
process.env.S3_SECRET_ACCESS_KEY = 'main-secret'
|
||||
process.env.AWS_ENDPOINT_URL_S3 = 'http://main-s3:9000'
|
||||
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 's3'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_BUCKET is required/)
|
||||
|
||||
process.env.ARCHIVE_S3_BUCKET = 'archive-bucket'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_ACCESS_KEY_ID is required/)
|
||||
|
||||
process.env.ARCHIVE_S3_ACCESS_KEY_ID = 'archive-key'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_SECRET_ACCESS_KEY is required/)
|
||||
|
||||
process.env.ARCHIVE_S3_SECRET_ACCESS_KEY = 'archive-secret'
|
||||
const cfg = getArchiveStorageConfig()
|
||||
expect(cfg).toMatchObject({
|
||||
enabled: true,
|
||||
backend: 's3',
|
||||
bucket: 'archive-bucket',
|
||||
endpoint: undefined,
|
||||
accessKeyId: 'archive-key',
|
||||
secretAccessKey: 'archive-secret',
|
||||
isSameS3Target: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('detects same-bucket S3 configuration and requires a non-STANDARD ARCHIVE_S3_STORAGE_CLASS', () => {
|
||||
process.env.STORAGE_BACKEND = 's3'
|
||||
process.env.S3_BUCKET = 'shared-bucket'
|
||||
process.env.AWS_ENDPOINT_URL_S3 = 'https://s3.example.com/'
|
||||
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 's3'
|
||||
process.env.ARCHIVE_S3_BUCKET = 'shared-bucket'
|
||||
process.env.ARCHIVE_AWS_ENDPOINT_URL_S3 = 'https://s3.example.com'
|
||||
process.env.ARCHIVE_S3_ACCESS_KEY_ID = 'ak'
|
||||
process.env.ARCHIVE_S3_SECRET_ACCESS_KEY = 'sk'
|
||||
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_STORAGE_CLASS must be set/)
|
||||
|
||||
process.env.ARCHIVE_S3_STORAGE_CLASS = 'standard'
|
||||
expect(() => getArchiveStorageConfig()).toThrow(/ARCHIVE_S3_STORAGE_CLASS must be set/)
|
||||
|
||||
process.env.ARCHIVE_S3_STORAGE_CLASS = 'GLACIER_IR'
|
||||
const cfg = getArchiveStorageConfig()
|
||||
expect(cfg).toMatchObject({
|
||||
enabled: true,
|
||||
backend: 's3',
|
||||
bucket: 'shared-bucket',
|
||||
storageClass: 'GLACIER_IR',
|
||||
isSameS3Target: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('LocalArchiveStorageService', () => {
|
||||
const testRoot = path.join(process.cwd(), 'data-test-archive-local')
|
||||
|
||||
beforeEach(() => {
|
||||
if (fs.existsSync(testRoot)) {
|
||||
fs.rmSync(testRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (fs.existsSync(testRoot)) {
|
||||
fs.rmSync(testRoot, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('stores files directly under <ARCHIVE_LOCAL_ROOT>/<key> without a bucket subfolder and cleans up empty parent folders on delete', async () => {
|
||||
const localArchive = new LocalArchiveStorageService(testRoot)
|
||||
const key = 'files/01M5P70NR77MCJRHDZK989S3H2/sample.mp4'
|
||||
const payload = 'raw video bytes content'
|
||||
|
||||
await localArchive.uploadStream(key, makeWebStream([payload]), payload.length, 'video/mp4')
|
||||
|
||||
const expectedDiskPath = path.join(
|
||||
testRoot,
|
||||
'files',
|
||||
'01M5P70NR77MCJRHDZK989S3H2',
|
||||
'sample.mp4',
|
||||
)
|
||||
expect(fs.existsSync(expectedDiskPath)).toBe(true)
|
||||
|
||||
const head = await localArchive.headObject(key)
|
||||
expect(head.size).toBe(payload.length)
|
||||
expect(head.key).toBe(key)
|
||||
|
||||
const { stream, contentLength } = await localArchive.getObjectStream(key)
|
||||
expect(contentLength).toBe(payload.length)
|
||||
const readBack = await readAllBytes(stream)
|
||||
expect(readBack.toString('utf8')).toBe(payload)
|
||||
|
||||
const restoreStatus = await localArchive.checkOrInitiateObjectRestore(key)
|
||||
expect(restoreStatus).toEqual({ status: 'ready' })
|
||||
|
||||
const deleted = await localArchive.deleteObject(key)
|
||||
expect(deleted).toBe(1)
|
||||
expect(fs.existsSync(expectedDiskPath)).toBe(false)
|
||||
// Empty <root>/files/<ulid> directory should also be cleaned up
|
||||
expect(fs.existsSync(path.dirname(expectedDiskPath))).toBe(false)
|
||||
})
|
||||
|
||||
it('prevents path traversal outside ARCHIVE_LOCAL_ROOT', async () => {
|
||||
const localArchive = new LocalArchiveStorageService(testRoot)
|
||||
await expect(
|
||||
localArchive.uploadStream('../escape.txt', makeWebStream(['bad']), 3),
|
||||
).rejects.toThrow(/path traversal/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('S3ArchiveStorageService', () => {
|
||||
const baseS3Config = {
|
||||
enabled: true as const,
|
||||
backend: 's3' as const,
|
||||
bucket: 'archive-bucket',
|
||||
endpoint: 'http://archive-s3:9000',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: 'archive-ak',
|
||||
secretAccessKey: 'archive-sk',
|
||||
forcePathStyle: true,
|
||||
storageClass: 'GLACIER_IR',
|
||||
useRestoreObject: 'auto' as const,
|
||||
restoreDays: 3,
|
||||
restoreTier: undefined,
|
||||
isSameS3Target: false,
|
||||
}
|
||||
|
||||
it('uploads small streams via PutObjectCommand with configured arbitrary StorageClass', async () => {
|
||||
s3SendSpy.mockResolvedValueOnce({})
|
||||
const service = new S3ArchiveStorageService({
|
||||
...baseS3Config,
|
||||
storageClass: 'ColdArchive',
|
||||
})
|
||||
|
||||
await service.uploadStream(
|
||||
'files/01ABC/video.mp4',
|
||||
makeWebStream(['hello archive']),
|
||||
13,
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(1)
|
||||
const cmd = s3SendSpy.mock.calls[0][0]
|
||||
expect(cmd).toBeInstanceOf(PutObjectCommand)
|
||||
expect(cmd.input).toMatchObject({
|
||||
Bucket: 'archive-bucket',
|
||||
Key: 'files/01ABC/video.mp4',
|
||||
ContentLength: 13,
|
||||
ContentType: 'video/mp4',
|
||||
StorageClass: 'ColdArchive',
|
||||
})
|
||||
})
|
||||
|
||||
it('streams large uploads via multipart upload and aborts on part failure', async () => {
|
||||
// Test multipart helper with custom thresholds (singlePutMaxBytes = 10, partSizeBytes = 5)
|
||||
const fakeClient = { send: s3SendSpy } as unknown as import('@aws-sdk/client-s3').S3Client
|
||||
s3SendSpy
|
||||
.mockResolvedValueOnce({ UploadId: 'mp-1' }) // CreateMultipartUpload
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
.mockResolvedValueOnce({ ETag: '"etag-1"' }) // Part 1
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
.mockResolvedValueOnce({ ETag: '"etag-2"' }) // Part 2
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
.mockResolvedValueOnce({ ETag: '"etag-3"' }) // Part 3
|
||||
.mockResolvedValueOnce({}) // CompleteMultipartUpload
|
||||
|
||||
await uploadStreamToS3Client(
|
||||
fakeClient,
|
||||
{
|
||||
bucket: 'archive-bucket',
|
||||
key: 'files/01ABC/large.bin',
|
||||
stream: makeWebStream(['01234', '56789', 'ab']),
|
||||
contentLength: 12,
|
||||
contentType: 'application/octet-stream',
|
||||
storageClass: 'DEEP_ARCHIVE',
|
||||
},
|
||||
5,
|
||||
10,
|
||||
)
|
||||
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(5)
|
||||
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(CreateMultipartUploadCommand)
|
||||
expect(s3SendSpy.mock.calls[0][0].input).toMatchObject({
|
||||
Bucket: 'archive-bucket',
|
||||
Key: 'files/01ABC/large.bin',
|
||||
StorageClass: 'DEEP_ARCHIVE',
|
||||
})
|
||||
expect(s3SendSpy.mock.calls[1][0]).toBeInstanceOf(UploadPartCommand)
|
||||
expect(s3SendSpy.mock.calls[2][0]).toBeInstanceOf(UploadPartCommand)
|
||||
expect(s3SendSpy.mock.calls[3][0]).toBeInstanceOf(UploadPartCommand)
|
||||
expect(s3SendSpy.mock.calls[4][0]).toBeInstanceOf(CompleteMultipartUploadCommand)
|
||||
expect(s3SendSpy.mock.calls[4][0].input.MultipartUpload.Parts).toEqual([
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
{ ETag: '"etag-1"', PartNumber: 1 },
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
{ ETag: '"etag-2"', PartNumber: 2 },
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
{ ETag: '"etag-3"', PartNumber: 3 },
|
||||
])
|
||||
|
||||
// Verify abort on failure
|
||||
s3SendSpy.mockReset()
|
||||
s3SendSpy
|
||||
.mockResolvedValueOnce({ UploadId: 'mp-fail' })
|
||||
.mockRejectedValueOnce(new Error('Part upload failed'))
|
||||
.mockResolvedValueOnce({}) // AbortMultipartUpload
|
||||
|
||||
await expect(
|
||||
uploadStreamToS3Client(
|
||||
fakeClient,
|
||||
{
|
||||
bucket: 'archive-bucket',
|
||||
key: 'files/01ABC/large.bin',
|
||||
stream: makeWebStream(['0123456789ab']),
|
||||
contentLength: 12,
|
||||
},
|
||||
5,
|
||||
10,
|
||||
),
|
||||
).rejects.toThrow('Part upload failed')
|
||||
|
||||
expect(s3SendSpy.mock.calls[2][0]).toBeInstanceOf(AbortMultipartUploadCommand)
|
||||
expect(s3SendSpy.mock.calls[2][0].input).toMatchObject({
|
||||
Bucket: 'archive-bucket',
|
||||
Key: 'files/01ABC/large.bin',
|
||||
UploadId: 'mp-fail',
|
||||
})
|
||||
})
|
||||
|
||||
it('handles checkOrInitiateObjectRestore in auto mode for synchronous vs asynchronous cold storage classes', async () => {
|
||||
// 1. GLACIER_IR is immediately readable -> ready without calling RestoreObjectCommand
|
||||
s3SendSpy.mockResolvedValueOnce({
|
||||
ContentLength: 100,
|
||||
StorageClass: 'GLACIER_IR',
|
||||
})
|
||||
const glacierIrService = new S3ArchiveStorageService({
|
||||
...baseS3Config,
|
||||
storageClass: 'GLACIER_IR',
|
||||
})
|
||||
expect(await glacierIrService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
|
||||
status: 'ready',
|
||||
})
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(1)
|
||||
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(HeadObjectCommand)
|
||||
|
||||
// 2. GLACIER with no Restore header -> sends RestoreObjectCommand without GlacierJobParameters by default
|
||||
s3SendSpy.mockReset()
|
||||
s3SendSpy
|
||||
.mockResolvedValueOnce({
|
||||
ContentLength: 100,
|
||||
StorageClass: 'GLACIER',
|
||||
})
|
||||
.mockResolvedValueOnce({})
|
||||
const glacierService = new S3ArchiveStorageService({
|
||||
...baseS3Config,
|
||||
storageClass: 'GLACIER',
|
||||
})
|
||||
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
|
||||
status: 'restoring',
|
||||
})
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(2)
|
||||
expect(s3SendSpy.mock.calls[1][0]).toBeInstanceOf(RestoreObjectCommand)
|
||||
expect(s3SendSpy.mock.calls[1][0].input).toEqual({
|
||||
Bucket: 'archive-bucket',
|
||||
Key: 'files/1/a.mp4',
|
||||
RestoreRequest: {
|
||||
Days: 3,
|
||||
},
|
||||
})
|
||||
|
||||
// 3. When ARCHIVE_S3_RESTORE_TIER is explicitly set, includes GlacierJobParameters
|
||||
s3SendSpy.mockReset()
|
||||
s3SendSpy
|
||||
.mockResolvedValueOnce({
|
||||
ContentLength: 100,
|
||||
StorageClass: 'DEEP_ARCHIVE',
|
||||
})
|
||||
.mockResolvedValueOnce({})
|
||||
const tieredService = new S3ArchiveStorageService({
|
||||
...baseS3Config,
|
||||
storageClass: 'DEEP_ARCHIVE',
|
||||
restoreTier: 'Standard',
|
||||
})
|
||||
expect(await tieredService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
|
||||
status: 'restoring',
|
||||
})
|
||||
expect(s3SendSpy.mock.calls[1][0].input).toEqual({
|
||||
Bucket: 'archive-bucket',
|
||||
Key: 'files/1/a.mp4',
|
||||
RestoreRequest: {
|
||||
Days: 3,
|
||||
GlacierJobParameters: { Tier: 'Standard' },
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Ongoing restore header -> returns 'restoring' without re-sending RestoreObject
|
||||
s3SendSpy.mockReset()
|
||||
s3SendSpy.mockResolvedValueOnce({
|
||||
ContentLength: 100,
|
||||
StorageClass: 'GLACIER',
|
||||
Restore: 'ongoing-request="true"',
|
||||
})
|
||||
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
|
||||
status: 'restoring',
|
||||
})
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(1)
|
||||
|
||||
// 5. Completed restore header -> returns 'ready'
|
||||
s3SendSpy.mockReset()
|
||||
s3SendSpy.mockResolvedValueOnce({
|
||||
ContentLength: 100,
|
||||
StorageClass: 'GLACIER',
|
||||
Restore: 'ongoing-request="false", expiry-date="Wed, 04 Nov 2026 00:00:00 GMT"',
|
||||
})
|
||||
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
|
||||
status: 'ready',
|
||||
})
|
||||
|
||||
// 6. RestoreAlreadyInProgress (409) is treated as idempotent success
|
||||
s3SendSpy.mockReset()
|
||||
const conflictErr = Object.assign(new Error('RestoreAlreadyInProgress'), {
|
||||
name: 'RestoreAlreadyInProgress',
|
||||
$metadata: { httpStatusCode: 409 },
|
||||
})
|
||||
s3SendSpy
|
||||
.mockResolvedValueOnce({
|
||||
ContentLength: 100,
|
||||
StorageClass: 'GLACIER',
|
||||
})
|
||||
.mockRejectedValueOnce(conflictErr)
|
||||
expect(await glacierService.checkOrInitiateObjectRestore('files/1/a.mp4')).toEqual({
|
||||
status: 'restoring',
|
||||
})
|
||||
})
|
||||
|
||||
it('rewrites StorageClass in-place via single CopyObject or multipart UploadPartCopy', async () => {
|
||||
const service = new S3ArchiveStorageService(
|
||||
{
|
||||
...baseS3Config,
|
||||
isSameS3Target: true,
|
||||
},
|
||||
{ singleCopyMaxBytes: 100, multipartCopyPartBytes: 60 },
|
||||
)
|
||||
|
||||
// Single CopyObject (<= 100 bytes)
|
||||
s3SendSpy.mockResolvedValueOnce({})
|
||||
await service.rewriteObjectStorageClassInPlace(
|
||||
'files/01ABC/my video #1.mp4',
|
||||
'GLACIER_IR',
|
||||
80,
|
||||
'video/mp4',
|
||||
)
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(1)
|
||||
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(CopyObjectCommand)
|
||||
expect(s3SendSpy.mock.calls[0][0].input).toEqual({
|
||||
Bucket: 'archive-bucket',
|
||||
Key: 'files/01ABC/my video #1.mp4',
|
||||
CopySource: 'archive-bucket/files/01ABC/my%20video%20%231.mp4',
|
||||
StorageClass: 'GLACIER_IR',
|
||||
MetadataDirective: 'COPY',
|
||||
ContentType: 'video/mp4',
|
||||
})
|
||||
|
||||
// Multipart UploadPartCopy (> 100 bytes: 130 bytes across 60-byte parts -> 0-59, 60-119, 120-129)
|
||||
s3SendSpy.mockReset()
|
||||
s3SendSpy
|
||||
.mockResolvedValueOnce({ UploadId: 'copy-mp-1' })
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
.mockResolvedValueOnce({ CopyPartResult: { ETag: '"p1"' } })
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
.mockResolvedValueOnce({ CopyPartResult: { ETag: '"p2"' } })
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
.mockResolvedValueOnce({ CopyPartResult: { ETag: '"p3"' } })
|
||||
.mockResolvedValueOnce({})
|
||||
|
||||
await service.rewriteObjectStorageClassInPlace(
|
||||
'files/01ABC/large.mp4',
|
||||
'STANDARD',
|
||||
130,
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
expect(s3SendSpy).toHaveBeenCalledTimes(5)
|
||||
expect(s3SendSpy.mock.calls[0][0]).toBeInstanceOf(CreateMultipartUploadCommand)
|
||||
expect(s3SendSpy.mock.calls[1][0]).toBeInstanceOf(UploadPartCopyCommand)
|
||||
expect(s3SendSpy.mock.calls[1][0].input.CopySourceRange).toBe('bytes=0-59')
|
||||
expect(s3SendSpy.mock.calls[2][0].input.CopySourceRange).toBe('bytes=60-119')
|
||||
expect(s3SendSpy.mock.calls[3][0].input.CopySourceRange).toBe('bytes=120-129')
|
||||
expect(s3SendSpy.mock.calls[4][0]).toBeInstanceOf(CompleteMultipartUploadCommand)
|
||||
})
|
||||
|
||||
it('detects cold storage restore errors and parses x-amz-restore headers accurately', () => {
|
||||
expect(parseRestoreHeaderStatus(undefined)).toBe('none')
|
||||
expect(parseRestoreHeaderStatus('ongoing-request="true"')).toBe('ongoing')
|
||||
expect(
|
||||
parseRestoreHeaderStatus(
|
||||
'ongoing-request="false", expiry-date="Fri, 21 Dec 2026 00:00:00 GMT"',
|
||||
),
|
||||
).toBe('restored')
|
||||
|
||||
expect(isColdStorageRestoreError({ name: 'InvalidObjectState' })).toBe(true)
|
||||
expect(isColdStorageRestoreError({ Code: 'RestoreAlreadyInProgress' })).toBe(true)
|
||||
expect(
|
||||
isColdStorageRestoreError(
|
||||
new Error("The operation is not valid for the object's storage class"),
|
||||
),
|
||||
).toBe(true)
|
||||
expect(isColdStorageRestoreError(new Error('NetworkingError'))).toBe(false)
|
||||
})
|
||||
|
||||
it('headObject maps 404/NotFound to NoSuchKey, but rethrows 500 and other transient errors', async () => {
|
||||
const service = new S3ArchiveStorageService(baseS3Config)
|
||||
|
||||
s3SendSpy.mockRejectedValueOnce({ name: 'NotFound', $metadata: { httpStatusCode: 404 } })
|
||||
await expect(service.headObject('missing.mp4')).rejects.toThrow('NoSuchKey')
|
||||
|
||||
const internalErr = new Error('Internal S3 500')
|
||||
s3SendSpy.mockRejectedValueOnce(internalErr)
|
||||
await expect(service.headObject('error.mp4')).rejects.toThrow('Internal S3 500')
|
||||
})
|
||||
|
||||
it('deleteObject returns 0 on 404/NotFound, but rethrows non-404 errors', async () => {
|
||||
const service = new S3ArchiveStorageService(baseS3Config)
|
||||
|
||||
s3SendSpy.mockRejectedValueOnce({ name: 'NotFound', $metadata: { httpStatusCode: 404 } })
|
||||
const res404 = await service.deleteObject('missing.mp4')
|
||||
expect(res404).toBe(0)
|
||||
|
||||
const permErr = new Error('AccessDenied')
|
||||
s3SendSpy.mockRejectedValueOnce(permErr)
|
||||
await expect(service.deleteObject('forbidden.mp4')).rejects.toThrow('AccessDenied')
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,735 @@
|
||||
import { ObjectInfo } from '@shumai/dtos'
|
||||
import {
|
||||
S3Client,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
CopyObjectCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCopyCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
AbortMultipartUploadCommand,
|
||||
RestoreObjectCommand,
|
||||
StorageClass,
|
||||
Tier,
|
||||
} from '@aws-sdk/client-s3'
|
||||
import * as crypto from 'crypto'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { Readable } from 'stream'
|
||||
import {
|
||||
bodyToWebReadableStream,
|
||||
isS3NotFoundError,
|
||||
S3StreamObject,
|
||||
uploadStreamToS3Client,
|
||||
writeStreamToLocalFile,
|
||||
} from './s3'
|
||||
|
||||
export type ArchiveStorageBackend = 'local' | 's3'
|
||||
export type ArchiveRestoreObjectMode = 'auto' | 'true' | 'false'
|
||||
|
||||
export interface ArchiveObjectInfo extends ObjectInfo {
|
||||
storageClass?: string
|
||||
restoreHeader?: string
|
||||
}
|
||||
|
||||
export interface ArchiveStorageConfigDisabled {
|
||||
enabled: false
|
||||
}
|
||||
|
||||
export interface ArchiveStorageConfigLocal {
|
||||
enabled: true
|
||||
backend: 'local'
|
||||
localRoot: string
|
||||
isSameS3Target: false
|
||||
}
|
||||
|
||||
export interface ArchiveStorageConfigS3 {
|
||||
enabled: true
|
||||
backend: 's3'
|
||||
bucket: string
|
||||
endpoint?: string
|
||||
region: string
|
||||
accessKeyId: string
|
||||
secretAccessKey: string
|
||||
forcePathStyle: boolean
|
||||
storageClass?: string
|
||||
useRestoreObject: ArchiveRestoreObjectMode
|
||||
restoreDays: number
|
||||
restoreTier?: string
|
||||
isSameS3Target: boolean
|
||||
}
|
||||
|
||||
export type ArchiveStorageConfig =
|
||||
ArchiveStorageConfigDisabled | ArchiveStorageConfigLocal | ArchiveStorageConfigS3
|
||||
|
||||
export interface ArchiveStorageService {
|
||||
readonly backend: ArchiveStorageBackend
|
||||
readonly isSameS3Target: boolean
|
||||
readonly storageClass?: string
|
||||
uploadStream: (
|
||||
key: string,
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
size: number,
|
||||
contentType?: string,
|
||||
) => Promise<void>
|
||||
getObjectStream: (key: string) => Promise<S3StreamObject>
|
||||
headObject: (key: string) => Promise<ArchiveObjectInfo>
|
||||
deleteObject: (key: string) => Promise<number>
|
||||
checkOrInitiateObjectRestore: (key: string) => Promise<{ status: 'ready' | 'restoring' }>
|
||||
initiateObjectRestoreExplicitly: (key: string) => Promise<{ status: 'ready' | 'restoring' }>
|
||||
rewriteObjectStorageClassInPlace: (
|
||||
key: string,
|
||||
targetStorageClass: string,
|
||||
expectedSize: number,
|
||||
contentType?: string,
|
||||
) => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage classes across AWS S3 and S3-compatible providers that require asynchronous
|
||||
* `RestoreObject` + `HeadObject` (`x-amz-restore`) polling before `GetObject` / `CopyObject`.
|
||||
* Explicitly excludes `GLACIER_IR` (Glacier Instant Retrieval), which is synchronously readable.
|
||||
*/
|
||||
export const KNOWN_ASYNC_RESTORE_STORAGE_CLASSES = new Set([
|
||||
'GLACIER',
|
||||
'DEEP_ARCHIVE',
|
||||
'ARCHIVE',
|
||||
'COLDARCHIVE',
|
||||
'COLD_ARCHIVE',
|
||||
'DEEPCOLDARCHIVE',
|
||||
'DEEP_COLD_ARCHIVE',
|
||||
'COLD',
|
||||
])
|
||||
|
||||
const SINGLE_COPY_MAX_BYTES = 5 * 1024 * 1024 * 1024 // 5 GB
|
||||
const MULTIPART_COPY_PART_BYTES = 512 * 1024 * 1024 // 512 MB
|
||||
|
||||
function normalizeEndpointForComparison(endpoint?: string): string {
|
||||
const trimmed = (endpoint || '').trim().toLowerCase().replace(/\/+$/, '')
|
||||
return trimmed || 'https://s3.amazonaws.com'
|
||||
}
|
||||
|
||||
function isSubPathOrEqual(candidatePath: string, targetPath: string): boolean {
|
||||
const rel = path.relative(targetPath, candidatePath)
|
||||
return rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))
|
||||
}
|
||||
|
||||
export function encodeCopySourceKey(bucket: string, key: string): string {
|
||||
const encodedKey = key
|
||||
.split('/')
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join('/')
|
||||
return `${bucket}/${encodedKey}`
|
||||
}
|
||||
|
||||
export function parseRestoreHeaderStatus(restoreHeader?: string): 'ongoing' | 'restored' | 'none' {
|
||||
if (!restoreHeader) return 'none'
|
||||
if (/ongoing-request\s*=\s*"true"/i.test(restoreHeader)) {
|
||||
return 'ongoing'
|
||||
}
|
||||
if (/ongoing-request\s*=\s*"false"/i.test(restoreHeader)) {
|
||||
return 'restored'
|
||||
}
|
||||
return 'none'
|
||||
}
|
||||
|
||||
export function isColdStorageRestoreError(err: unknown): boolean {
|
||||
if (!err || typeof err !== 'object') return false
|
||||
const e = err as Record<string, unknown>
|
||||
const code =
|
||||
(typeof e.name === 'string' && e.name) ||
|
||||
(typeof e.Code === 'string' && e.Code) ||
|
||||
(typeof e.code === 'string' && e.code) ||
|
||||
''
|
||||
if (code === 'InvalidObjectState' || code === 'RestoreAlreadyInProgress') {
|
||||
return true
|
||||
}
|
||||
const msg = typeof e.message === 'string' ? e.message : ''
|
||||
return (
|
||||
msg.includes('InvalidObjectState') ||
|
||||
msg.includes('RestoreAlreadyInProgress') ||
|
||||
msg.includes("operation is not valid for the object's storage class")
|
||||
)
|
||||
}
|
||||
|
||||
function isRestoreAlreadyInProgressError(err: unknown): boolean {
|
||||
if (!err || typeof err !== 'object') return false
|
||||
const e = err as Record<string, unknown>
|
||||
const metadata = e.$metadata as { httpStatusCode?: number } | undefined
|
||||
const code =
|
||||
(typeof e.name === 'string' && e.name) ||
|
||||
(typeof e.Code === 'string' && e.Code) ||
|
||||
(typeof e.code === 'string' && e.code) ||
|
||||
''
|
||||
if (code === 'RestoreAlreadyInProgress' || metadata?.httpStatusCode === 409) {
|
||||
return true
|
||||
}
|
||||
return typeof e.message === 'string' && e.message.includes('RestoreAlreadyInProgress')
|
||||
}
|
||||
|
||||
export function getArchiveStorageConfig(): ArchiveStorageConfig {
|
||||
const rawBackend = (process.env.ARCHIVE_STORAGE_BACKEND || '').trim()
|
||||
if (!rawBackend) {
|
||||
return { enabled: false }
|
||||
}
|
||||
|
||||
if (rawBackend !== 'local' && rawBackend !== 's3') {
|
||||
throw new Error(
|
||||
`Invalid ARCHIVE_STORAGE_BACKEND "${rawBackend}". Expected "local", "s3", or empty.`,
|
||||
)
|
||||
}
|
||||
|
||||
if (rawBackend === 'local') {
|
||||
const rawLocalRoot = (process.env.ARCHIVE_LOCAL_ROOT || '').trim()
|
||||
if (!rawLocalRoot) {
|
||||
throw new Error(
|
||||
'ARCHIVE_LOCAL_ROOT is required when ARCHIVE_STORAGE_BACKEND is set to "local".',
|
||||
)
|
||||
}
|
||||
const resolvedArchiveRoot = path.resolve(rawLocalRoot)
|
||||
const mainBackend = (process.env.STORAGE_BACKEND || 'local').trim()
|
||||
if (mainBackend !== 's3') {
|
||||
const mainBaseDir = path.resolve('data')
|
||||
const mainBucketDir = path.resolve('data', process.env.S3_BUCKET || 'shumai')
|
||||
if (
|
||||
resolvedArchiveRoot === mainBaseDir ||
|
||||
isSubPathOrEqual(resolvedArchiveRoot, mainBucketDir) ||
|
||||
isSubPathOrEqual(mainBucketDir, resolvedArchiveRoot)
|
||||
) {
|
||||
throw new Error(
|
||||
`ARCHIVE_LOCAL_ROOT (${resolvedArchiveRoot}) must not overlap with main local storage (${mainBucketDir}).`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: true,
|
||||
backend: 'local',
|
||||
localRoot: resolvedArchiveRoot,
|
||||
isSameS3Target: false,
|
||||
}
|
||||
}
|
||||
|
||||
// S3 backend — never fall back to main S3_* environment variables
|
||||
const bucket = (process.env.ARCHIVE_S3_BUCKET || '').trim()
|
||||
if (!bucket) {
|
||||
throw new Error('ARCHIVE_S3_BUCKET is required when ARCHIVE_STORAGE_BACKEND is set to "s3".')
|
||||
}
|
||||
|
||||
const accessKeyId = (process.env.ARCHIVE_S3_ACCESS_KEY_ID || '').trim()
|
||||
if (!accessKeyId) {
|
||||
throw new Error(
|
||||
'ARCHIVE_S3_ACCESS_KEY_ID is required when ARCHIVE_STORAGE_BACKEND is set to "s3".',
|
||||
)
|
||||
}
|
||||
|
||||
const secretAccessKey = (process.env.ARCHIVE_S3_SECRET_ACCESS_KEY || '').trim()
|
||||
if (!secretAccessKey) {
|
||||
throw new Error(
|
||||
'ARCHIVE_S3_SECRET_ACCESS_KEY is required when ARCHIVE_STORAGE_BACKEND is set to "s3".',
|
||||
)
|
||||
}
|
||||
|
||||
const endpoint = (process.env.ARCHIVE_AWS_ENDPOINT_URL_S3 || '').trim() || undefined
|
||||
const region = (process.env.ARCHIVE_S3_REGION || '').trim() || 'auto'
|
||||
const forcePathStyleRaw = (process.env.ARCHIVE_S3_FORCE_PATH_STYLE || 'true').trim().toLowerCase()
|
||||
const forcePathStyle = forcePathStyleRaw !== 'false' && forcePathStyleRaw !== '0'
|
||||
const storageClass = (process.env.ARCHIVE_S3_STORAGE_CLASS || '').trim() || undefined
|
||||
|
||||
const useRestoreObjectRaw = (process.env.ARCHIVE_S3_USE_RESTORE_OBJECT || 'auto')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
if (
|
||||
useRestoreObjectRaw !== 'auto' &&
|
||||
useRestoreObjectRaw !== 'true' &&
|
||||
useRestoreObjectRaw !== 'false'
|
||||
) {
|
||||
throw new Error(
|
||||
`Invalid ARCHIVE_S3_USE_RESTORE_OBJECT "${useRestoreObjectRaw}". Expected "auto", "true", or "false".`,
|
||||
)
|
||||
}
|
||||
|
||||
const restoreDaysRaw = (process.env.ARCHIVE_S3_RESTORE_DAYS || '1').trim()
|
||||
const restoreDays = Number.parseInt(restoreDaysRaw, 10)
|
||||
if (!Number.isFinite(restoreDays) || restoreDays < 1) {
|
||||
throw new Error(
|
||||
`Invalid ARCHIVE_S3_RESTORE_DAYS "${restoreDaysRaw}". Expected a positive integer.`,
|
||||
)
|
||||
}
|
||||
|
||||
const restoreTier = (process.env.ARCHIVE_S3_RESTORE_TIER || '').trim() || undefined
|
||||
|
||||
const mainBackend = (process.env.STORAGE_BACKEND || 'local').trim()
|
||||
const mainBucket = (process.env.S3_BUCKET || 'shumai').trim()
|
||||
const mainEndpoint = (process.env.AWS_ENDPOINT_URL_S3 || '').trim() || undefined
|
||||
|
||||
const isSameS3Target =
|
||||
mainBackend === 's3' &&
|
||||
mainBucket === bucket &&
|
||||
normalizeEndpointForComparison(mainEndpoint) === normalizeEndpointForComparison(endpoint)
|
||||
|
||||
if (isSameS3Target) {
|
||||
if (!storageClass || storageClass.toUpperCase() === 'STANDARD') {
|
||||
throw new Error(
|
||||
'When ARCHIVE_S3_BUCKET and endpoint match main S3 storage, ARCHIVE_S3_STORAGE_CLASS must be set to a non-STANDARD storage class.',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: true,
|
||||
backend: 's3',
|
||||
bucket,
|
||||
endpoint,
|
||||
region,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
forcePathStyle,
|
||||
storageClass,
|
||||
useRestoreObject: useRestoreObjectRaw,
|
||||
restoreDays,
|
||||
restoreTier,
|
||||
isSameS3Target,
|
||||
}
|
||||
}
|
||||
|
||||
export function isArchiveStorageEnabled(): boolean {
|
||||
const rawBackend = (process.env.ARCHIVE_STORAGE_BACKEND || '').trim()
|
||||
return rawBackend === 'local' || rawBackend === 's3'
|
||||
}
|
||||
|
||||
export function validateArchiveStorageConfig(): ArchiveStorageConfig {
|
||||
return getArchiveStorageConfig()
|
||||
}
|
||||
|
||||
export function getArchiveStoragePath(
|
||||
key: string,
|
||||
): { path: string; backend: 's3' | 'local' } | null {
|
||||
const config = getArchiveStorageConfig()
|
||||
if (!config.enabled) return null
|
||||
if (config.backend === 's3') {
|
||||
return { path: `s3://${config.bucket}/${key}`, backend: 's3' }
|
||||
}
|
||||
return { path: path.join(config.localRoot, key), backend: 'local' }
|
||||
}
|
||||
|
||||
export class LocalArchiveStorageService implements ArchiveStorageService {
|
||||
readonly backend = 'local' as const
|
||||
readonly isSameS3Target = false
|
||||
readonly storageClass = undefined
|
||||
private readonly rootPath: string
|
||||
|
||||
constructor(rootPath: string) {
|
||||
this.rootPath = path.resolve(rootPath)
|
||||
if (!fs.existsSync(this.rootPath)) {
|
||||
fs.mkdirSync(this.rootPath, { recursive: true })
|
||||
}
|
||||
}
|
||||
|
||||
private getFilePath(key: string): string {
|
||||
const normalizedKey = key.replace(/^\/+/, '')
|
||||
if (!normalizedKey) {
|
||||
throw new Error('Invalid archive key: key cannot be empty')
|
||||
}
|
||||
const filePath = path.resolve(this.rootPath, normalizedKey)
|
||||
if (!isSubPathOrEqual(filePath, this.rootPath) || filePath === this.rootPath) {
|
||||
throw new Error('Invalid archive path: potential path traversal detected')
|
||||
}
|
||||
return filePath
|
||||
}
|
||||
|
||||
async uploadStream(
|
||||
key: string,
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
_size: number,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
_contentType?: string,
|
||||
): Promise<void> {
|
||||
const filePath = this.getFilePath(key)
|
||||
await writeStreamToLocalFile(filePath, stream)
|
||||
}
|
||||
|
||||
async getObjectStream(key: string): Promise<S3StreamObject> {
|
||||
const filePath = this.getFilePath(key)
|
||||
try {
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
const nodeStream = fs.createReadStream(filePath)
|
||||
return {
|
||||
stream: Readable.toWeb(nodeStream) as unknown as ReadableStream<Uint8Array>,
|
||||
contentLength: stats.size,
|
||||
contentType: Bun.file(filePath).type || 'application/octet-stream',
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
throw new Error('NoSuchKey: The specified key does not exist.', { cause: e })
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async headObject(key: string): Promise<ArchiveObjectInfo> {
|
||||
const filePath = this.getFilePath(key)
|
||||
try {
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
const hash = crypto
|
||||
.createHash('md5')
|
||||
.update(filePath + stats.mtimeMs)
|
||||
.digest('hex')
|
||||
return {
|
||||
key,
|
||||
size: stats.size,
|
||||
lastModified: stats.mtime,
|
||||
contentType: Bun.file(key).type || 'application/octet-stream',
|
||||
eTag: `"${hash}"`,
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
throw new Error('NoSuchKey: The specified key does not exist.', { cause: e })
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async deleteObject(key: string): Promise<number> {
|
||||
const filePath = this.getFilePath(key)
|
||||
try {
|
||||
await fs.promises.unlink(filePath)
|
||||
// Best-effort cleanup of empty parent folder (e.g. `<root>/files/<ulid>`)
|
||||
const parentDir = path.dirname(filePath)
|
||||
if (parentDir !== this.rootPath && isSubPathOrEqual(parentDir, this.rootPath)) {
|
||||
const remaining = await fs.promises.readdir(parentDir).catch(() => ['non-empty'])
|
||||
if (remaining.length === 0) {
|
||||
await fs.promises.rmdir(parentDir).catch(() => {})
|
||||
}
|
||||
}
|
||||
return 1
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return 0
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async checkOrInitiateObjectRestore(_key: string): Promise<{ status: 'ready' | 'restoring' }> {
|
||||
return { status: 'ready' }
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async initiateObjectRestoreExplicitly(_key: string): Promise<{ status: 'ready' | 'restoring' }> {
|
||||
return { status: 'ready' }
|
||||
}
|
||||
|
||||
async rewriteObjectStorageClassInPlace(): Promise<void> {
|
||||
throw new Error('rewriteObjectStorageClassInPlace is only supported for S3 archive storage.')
|
||||
}
|
||||
}
|
||||
|
||||
export class S3ArchiveStorageService implements ArchiveStorageService {
|
||||
readonly backend = 's3' as const
|
||||
readonly isSameS3Target: boolean
|
||||
readonly storageClass?: string
|
||||
private readonly client: S3Client
|
||||
private readonly bucket: string
|
||||
private readonly useRestoreObject: ArchiveRestoreObjectMode
|
||||
private readonly restoreDays: number
|
||||
private readonly restoreTier?: string
|
||||
private readonly singleCopyMaxBytes: number
|
||||
private readonly multipartCopyPartBytes: number
|
||||
|
||||
constructor(
|
||||
config: ArchiveStorageConfigS3,
|
||||
options?: { singleCopyMaxBytes?: number; multipartCopyPartBytes?: number },
|
||||
) {
|
||||
this.bucket = config.bucket
|
||||
this.isSameS3Target = config.isSameS3Target
|
||||
this.storageClass = config.storageClass
|
||||
this.useRestoreObject = config.useRestoreObject
|
||||
this.restoreDays = config.restoreDays
|
||||
this.restoreTier = config.restoreTier
|
||||
this.singleCopyMaxBytes = options?.singleCopyMaxBytes ?? SINGLE_COPY_MAX_BYTES
|
||||
this.multipartCopyPartBytes = options?.multipartCopyPartBytes ?? MULTIPART_COPY_PART_BYTES
|
||||
|
||||
this.client = new S3Client({
|
||||
region: config.region,
|
||||
...(config.endpoint ? { endpoint: config.endpoint } : {}),
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
},
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
requestChecksumCalculation: 'WHEN_REQUIRED',
|
||||
responseChecksumValidation: 'WHEN_REQUIRED',
|
||||
})
|
||||
}
|
||||
|
||||
async uploadStream(
|
||||
key: string,
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
size: number,
|
||||
contentType?: string,
|
||||
): Promise<void> {
|
||||
await uploadStreamToS3Client(this.client, {
|
||||
bucket: this.bucket,
|
||||
key,
|
||||
stream,
|
||||
contentLength: size,
|
||||
contentType,
|
||||
storageClass: this.storageClass,
|
||||
})
|
||||
}
|
||||
|
||||
async getObjectStream(key: string): Promise<S3StreamObject> {
|
||||
const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }))
|
||||
return {
|
||||
stream: bodyToWebReadableStream(res.Body),
|
||||
contentLength: res.ContentLength,
|
||||
contentType: res.ContentType || Bun.file(key).type || 'application/octet-stream',
|
||||
}
|
||||
}
|
||||
|
||||
async headObject(key: string): Promise<ArchiveObjectInfo> {
|
||||
try {
|
||||
const res = await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key }))
|
||||
return {
|
||||
key,
|
||||
size: res.ContentLength ?? 0,
|
||||
lastModified: res.LastModified ?? new Date(),
|
||||
contentType: res.ContentType || 'application/octet-stream',
|
||||
eTag: res.ETag || '',
|
||||
storageClass: res.StorageClass,
|
||||
restoreHeader: res.Restore,
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
if (isS3NotFoundError(err)) {
|
||||
throw new Error('NoSuchKey: The specified key does not exist.', { cause: err })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
async deleteObject(key: string): Promise<number> {
|
||||
try {
|
||||
await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: key }))
|
||||
return 1
|
||||
} catch (err: unknown) {
|
||||
if (isS3NotFoundError(err)) {
|
||||
return 0
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
private async sendRestoreObjectRequest(key: string): Promise<void> {
|
||||
try {
|
||||
await this.client.send(
|
||||
new RestoreObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
RestoreRequest: {
|
||||
Days: this.restoreDays,
|
||||
...(this.restoreTier
|
||||
? { GlacierJobParameters: { Tier: this.restoreTier as Tier } }
|
||||
: {}),
|
||||
},
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
}),
|
||||
)
|
||||
} catch (err: unknown) {
|
||||
if (isRestoreAlreadyInProgressError(err)) {
|
||||
return
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
async checkOrInitiateObjectRestore(key: string): Promise<{ status: 'ready' | 'restoring' }> {
|
||||
if (this.useRestoreObject === 'false') {
|
||||
return { status: 'ready' }
|
||||
}
|
||||
|
||||
const head = await this.headObject(key)
|
||||
const restoreState = parseRestoreHeaderStatus(head.restoreHeader)
|
||||
if (restoreState === 'restored') {
|
||||
return { status: 'ready' }
|
||||
}
|
||||
if (restoreState === 'ongoing') {
|
||||
return { status: 'restoring' }
|
||||
}
|
||||
|
||||
const effectiveStorageClass = (head.storageClass || this.storageClass || '').toUpperCase()
|
||||
const requiresRestore =
|
||||
this.useRestoreObject === 'true' ||
|
||||
(this.useRestoreObject === 'auto' &&
|
||||
KNOWN_ASYNC_RESTORE_STORAGE_CLASSES.has(effectiveStorageClass))
|
||||
|
||||
if (!requiresRestore) {
|
||||
return { status: 'ready' }
|
||||
}
|
||||
|
||||
await this.sendRestoreObjectRequest(key)
|
||||
return { status: 'restoring' }
|
||||
}
|
||||
|
||||
async initiateObjectRestoreExplicitly(key: string): Promise<{ status: 'ready' | 'restoring' }> {
|
||||
if (this.useRestoreObject === 'false') {
|
||||
throw new Error(
|
||||
'Object is in a cold storage class requiring RestoreObject, but ARCHIVE_S3_USE_RESTORE_OBJECT is set to "false".',
|
||||
)
|
||||
}
|
||||
const head = await this.headObject(key)
|
||||
const restoreState = parseRestoreHeaderStatus(head.restoreHeader)
|
||||
if (restoreState === 'restored') {
|
||||
return { status: 'ready' }
|
||||
}
|
||||
if (restoreState === 'ongoing') {
|
||||
return { status: 'restoring' }
|
||||
}
|
||||
await this.sendRestoreObjectRequest(key)
|
||||
return { status: 'restoring' }
|
||||
}
|
||||
|
||||
async rewriteObjectStorageClassInPlace(
|
||||
key: string,
|
||||
targetStorageClass: string,
|
||||
expectedSize: number,
|
||||
contentType?: string,
|
||||
): Promise<void> {
|
||||
const copySource = encodeCopySourceKey(this.bucket, key)
|
||||
const s3StorageClass = targetStorageClass as StorageClass
|
||||
|
||||
if (expectedSize <= this.singleCopyMaxBytes) {
|
||||
await this.client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
CopySource: copySource,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
StorageClass: s3StorageClass,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
MetadataDirective: 'COPY',
|
||||
...(contentType ? { ContentType: contentType } : {}),
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const createRes = await this.client.send(
|
||||
new CreateMultipartUploadCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
StorageClass: s3StorageClass,
|
||||
...(contentType ? { ContentType: contentType } : {}),
|
||||
}),
|
||||
)
|
||||
const uploadId = createRes.UploadId
|
||||
if (!uploadId) {
|
||||
throw new Error(`Failed to initiate multipart copy for ${this.bucket}/${key}`)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const completedParts: Array<{ ETag?: string; PartNumber: number }> = []
|
||||
let partNumber = 1
|
||||
let offset = 0
|
||||
|
||||
try {
|
||||
while (offset < expectedSize) {
|
||||
const end = Math.min(offset + this.multipartCopyPartBytes - 1, expectedSize - 1)
|
||||
const currentPartNumber = partNumber++
|
||||
const copyPartRes = await this.client.send(
|
||||
new UploadPartCopyCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: currentPartNumber,
|
||||
CopySource: copySource,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
CopySourceRange: `bytes=${offset}-${end}`,
|
||||
}),
|
||||
)
|
||||
completedParts.push({
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
ETag: copyPartRes.CopyPartResult?.ETag,
|
||||
PartNumber: currentPartNumber,
|
||||
})
|
||||
offset = end + 1
|
||||
}
|
||||
|
||||
await this.client.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
MultipartUpload: {
|
||||
Parts: completedParts,
|
||||
},
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
}),
|
||||
)
|
||||
} catch (err) {
|
||||
await this.client
|
||||
.send(
|
||||
new AbortMultipartUploadCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
}),
|
||||
)
|
||||
.catch(() => {})
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let cachedService: ArchiveStorageService | null = null
|
||||
let cachedConfigSignature: string | null = null
|
||||
|
||||
function getEnvSignature(): string {
|
||||
return JSON.stringify({
|
||||
b: process.env.ARCHIVE_STORAGE_BACKEND,
|
||||
lr: process.env.ARCHIVE_LOCAL_ROOT,
|
||||
sb: process.env.ARCHIVE_S3_BUCKET,
|
||||
se: process.env.ARCHIVE_AWS_ENDPOINT_URL_S3,
|
||||
sr: process.env.ARCHIVE_S3_REGION,
|
||||
ak: process.env.ARCHIVE_S3_ACCESS_KEY_ID,
|
||||
sk: process.env.ARCHIVE_S3_SECRET_ACCESS_KEY,
|
||||
fp: process.env.ARCHIVE_S3_FORCE_PATH_STYLE,
|
||||
sc: process.env.ARCHIVE_S3_STORAGE_CLASS,
|
||||
ro: process.env.ARCHIVE_S3_USE_RESTORE_OBJECT,
|
||||
rd: process.env.ARCHIVE_S3_RESTORE_DAYS,
|
||||
rt: process.env.ARCHIVE_S3_RESTORE_TIER,
|
||||
mb: process.env.STORAGE_BACKEND,
|
||||
msb: process.env.S3_BUCKET,
|
||||
mse: process.env.AWS_ENDPOINT_URL_S3,
|
||||
})
|
||||
}
|
||||
|
||||
export function getArchiveStorageService(): ArchiveStorageService {
|
||||
const sig = getEnvSignature()
|
||||
if (cachedService && cachedConfigSignature === sig) {
|
||||
return cachedService
|
||||
}
|
||||
|
||||
const config = getArchiveStorageConfig()
|
||||
if (!config.enabled) {
|
||||
throw new Error('Archive storage is not enabled (ARCHIVE_STORAGE_BACKEND is not configured).')
|
||||
}
|
||||
|
||||
cachedService =
|
||||
config.backend === 'local'
|
||||
? new LocalArchiveStorageService(config.localRoot)
|
||||
: new S3ArchiveStorageService(config)
|
||||
cachedConfigSignature = sig
|
||||
return cachedService
|
||||
}
|
||||
|
||||
export function resetArchiveStorageServiceForTests(): void {
|
||||
cachedService = null
|
||||
cachedConfigSignature = null
|
||||
}
|
||||
@@ -10,7 +10,13 @@ import {
|
||||
ListPartsCommand,
|
||||
AbortMultipartUploadCommand,
|
||||
} from '@aws-sdk/client-s3'
|
||||
import { buildContentDisposition, LocalStorageService, S3StorageService } from './s3'
|
||||
import {
|
||||
buildContentDisposition,
|
||||
LocalStorageService,
|
||||
S3StorageService,
|
||||
signLocalGetUrl,
|
||||
verifyLocalGetUrlSignature,
|
||||
} from './s3'
|
||||
|
||||
const s3ClientConstructorSpy = vi.fn()
|
||||
const s3SendSpy = vi.fn()
|
||||
@@ -40,6 +46,7 @@ describe('S3Service implementations', () => {
|
||||
vi.clearAllMocks()
|
||||
getSignedUrlSpy.mockReset()
|
||||
delete process.env.PRESIGNED_URL_EXPIRES_IN
|
||||
delete process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN
|
||||
getSignedUrlSpy.mockResolvedValue('http://presigned-url')
|
||||
s3SendSpy.mockResolvedValue({})
|
||||
})
|
||||
@@ -146,7 +153,7 @@ describe('S3Service implementations', () => {
|
||||
expect(getSignedUrlSpy).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('should respect PRESIGNED_URL_EXPIRES_IN env', async () => {
|
||||
it('should respect PRESIGNED_URL_EXPIRES_IN env for GET', async () => {
|
||||
process.env.PRESIGNED_URL_EXPIRES_IN = '10'
|
||||
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
|
||||
|
||||
@@ -159,6 +166,77 @@ describe('S3Service implementations', () => {
|
||||
expect.objectContaining({ expiresIn: 10 * 3600 }),
|
||||
)
|
||||
|
||||
delete process.env.PRESIGNED_URL_EXPIRES_IN
|
||||
delete process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN
|
||||
})
|
||||
|
||||
it('should default PUT presign and multipart to 12 hours (43200 seconds)', async () => {
|
||||
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
|
||||
|
||||
getSignedUrlSpy.mockClear()
|
||||
await s3.presign('bucket', 'key', 'PUT')
|
||||
expect(getSignedUrlSpy).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(PutObjectCommand),
|
||||
expect.objectContaining({ expiresIn: 12 * 3600 }),
|
||||
)
|
||||
|
||||
getSignedUrlSpy.mockClear()
|
||||
await s3.presignMultipart('bucket', 'key', {
|
||||
method: 'PUT',
|
||||
key: 'key',
|
||||
fileId: 'file-1',
|
||||
uploadId: 'up-1',
|
||||
partNumber: 1,
|
||||
})
|
||||
expect(getSignedUrlSpy).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(UploadPartCommand),
|
||||
expect.objectContaining({ expiresIn: 12 * 3600 }),
|
||||
)
|
||||
})
|
||||
|
||||
it('should respect PRESIGNED_UPLOAD_URL_EXPIRES_IN env for PUT and multipart', async () => {
|
||||
process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN = '24'
|
||||
process.env.PRESIGNED_URL_EXPIRES_IN = '5'
|
||||
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
|
||||
|
||||
getSignedUrlSpy.mockClear()
|
||||
await s3.presign('bucket', 'key', 'PUT')
|
||||
expect(getSignedUrlSpy).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(PutObjectCommand),
|
||||
expect.objectContaining({ expiresIn: 24 * 3600 }),
|
||||
)
|
||||
|
||||
getSignedUrlSpy.mockClear()
|
||||
await s3.presignMultipart('bucket', 'key', {
|
||||
method: 'PUT',
|
||||
key: 'key',
|
||||
fileId: 'file-1',
|
||||
})
|
||||
expect(getSignedUrlSpy).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(PutObjectCommand),
|
||||
expect.objectContaining({ expiresIn: 24 * 3600 }),
|
||||
)
|
||||
|
||||
delete process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN
|
||||
delete process.env.PRESIGNED_URL_EXPIRES_IN
|
||||
})
|
||||
|
||||
it('should allow PRESIGNED_URL_EXPIRES_IN to override PUT if PRESIGNED_UPLOAD_URL_EXPIRES_IN is unset', async () => {
|
||||
process.env.PRESIGNED_URL_EXPIRES_IN = '8'
|
||||
const s3 = new S3StorageService('http://localhost:9000', 'key', 'secret', 'test-bucket')
|
||||
|
||||
getSignedUrlSpy.mockClear()
|
||||
await s3.presign('bucket', 'key', 'PUT')
|
||||
expect(getSignedUrlSpy).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(PutObjectCommand),
|
||||
expect.objectContaining({ expiresIn: 8 * 3600 }),
|
||||
)
|
||||
|
||||
delete process.env.PRESIGNED_URL_EXPIRES_IN
|
||||
})
|
||||
|
||||
@@ -638,19 +716,25 @@ describe('S3Service implementations', () => {
|
||||
|
||||
it('should generate a local presign URL', async () => {
|
||||
const url = await localS3.presign('my-bucket', 'dir/file.txt', 'GET')
|
||||
expect(url).toBe('http://localhost:3000/files/my-bucket/dir/file.txt')
|
||||
expect(url).toContain('http://localhost:3000/files/my-bucket/dir/file.txt?')
|
||||
expect(url).toContain('Signature=')
|
||||
expect(url).toContain('Expires=')
|
||||
})
|
||||
|
||||
it('should generate a local presign URL with download param', async () => {
|
||||
const url = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true)
|
||||
expect(url).toBe('http://localhost:3000/files/my-bucket/dir/file.txt?download=1')
|
||||
expect(url).toContain('http://localhost:3000/files/my-bucket/dir/file.txt?')
|
||||
expect(url).toContain('Signature=')
|
||||
expect(url).toContain('Expires=')
|
||||
expect(url).toContain('download=1')
|
||||
})
|
||||
|
||||
it('should include an encoded filename in the local download URL', async () => {
|
||||
const url = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true, 'foo bar.png')
|
||||
expect(url).toBe(
|
||||
'http://localhost:3000/files/my-bucket/dir/file.txt?download=1&filename=foo%20bar.png',
|
||||
)
|
||||
expect(url).toContain('http://localhost:3000/files/my-bucket/dir/file.txt?')
|
||||
expect(url).toContain('Signature=')
|
||||
expect(url).toContain('Expires=')
|
||||
expect(url).toContain('download=1&filename=foo%20bar.png')
|
||||
})
|
||||
|
||||
it('should not append a filename param when none is provided', async () => {
|
||||
@@ -658,6 +742,23 @@ describe('S3Service implementations', () => {
|
||||
expect(url).not.toContain('filename=')
|
||||
})
|
||||
|
||||
it('should cache GET presigned URLs using LRU cache', async () => {
|
||||
const url1 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET')
|
||||
const url2 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET')
|
||||
expect(url1).toBe(url2)
|
||||
|
||||
const urlOther = await localS3.presign('my-bucket', 'dir/other.txt', 'GET')
|
||||
expect(urlOther).not.toBe(url1)
|
||||
})
|
||||
|
||||
it('should not cache download URLs', async () => {
|
||||
const url1 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true)
|
||||
const url2 = await localS3.presign('my-bucket', 'dir/file.txt', 'GET', true)
|
||||
// download URLs include download=1
|
||||
expect(url1).toContain('download=1')
|
||||
expect(url2).toContain('download=1')
|
||||
})
|
||||
|
||||
it('should throw an error for unsupported presign methods', async () => {
|
||||
await expect(localS3.presign('b', 'k', 'POST')).rejects.toThrow()
|
||||
})
|
||||
@@ -670,7 +771,9 @@ describe('S3Service implementations', () => {
|
||||
|
||||
it('should resolveInput to presigned URL if file does not exist on disk', async () => {
|
||||
const input = await localS3.resolveInput('my-bucket', 'nonexistent.mp4')
|
||||
expect(input).toBe('http://localhost:3000/files/my-bucket/nonexistent.mp4')
|
||||
expect(input).toContain('http://localhost:3000/files/my-bucket/nonexistent.mp4?')
|
||||
expect(input).toContain('Signature=')
|
||||
expect(input).toContain('Expires=')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -760,4 +863,63 @@ describe('S3Service implementations', () => {
|
||||
).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('signLocalGetUrl and verifyLocalGetUrlSignature', () => {
|
||||
it('signs and verifies valid local get url', () => {
|
||||
const { signature, expires } = signLocalGetUrl('my-bucket', 'files/123/file.mp4')
|
||||
expect(signature).toBeDefined()
|
||||
expect(expires).toBeGreaterThan(Math.floor(Date.now() / 1000))
|
||||
|
||||
const isValid = verifyLocalGetUrlSignature(
|
||||
'my-bucket',
|
||||
'files/123/file.mp4',
|
||||
signature,
|
||||
String(expires),
|
||||
)
|
||||
expect(isValid).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects expired local get url signature', () => {
|
||||
// 10 seconds in the past
|
||||
const { signature, expires } = signLocalGetUrl('my-bucket', 'files/123/file.mp4', -10)
|
||||
const isValid = verifyLocalGetUrlSignature(
|
||||
'my-bucket',
|
||||
'files/123/file.mp4',
|
||||
signature,
|
||||
String(expires),
|
||||
)
|
||||
expect(isValid).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects tampered bucket or key', () => {
|
||||
const { signature, expires } = signLocalGetUrl('my-bucket', 'files/123/file.mp4')
|
||||
expect(
|
||||
verifyLocalGetUrlSignature(
|
||||
'other-bucket',
|
||||
'files/123/file.mp4',
|
||||
signature,
|
||||
String(expires),
|
||||
),
|
||||
).toBe(false)
|
||||
|
||||
expect(
|
||||
verifyLocalGetUrlSignature('my-bucket', 'files/999/other.mp4', signature, String(expires)),
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects tampered or malformed signature or expires', () => {
|
||||
expect(
|
||||
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', 'badsig', '1234567890'),
|
||||
).toBe(false)
|
||||
expect(
|
||||
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', undefined, '1234567890'),
|
||||
).toBe(false)
|
||||
expect(
|
||||
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', 'badsig', undefined),
|
||||
).toBe(false)
|
||||
expect(
|
||||
verifyLocalGetUrlSignature('my-bucket', 'files/123/file.mp4', 'badsig', 'not-a-number'),
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
+372
-9
@@ -14,6 +14,7 @@ import {
|
||||
ListPartsCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
AbortMultipartUploadCommand,
|
||||
StorageClass,
|
||||
} from '@aws-sdk/client-s3'
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
|
||||
import * as crypto from 'crypto'
|
||||
@@ -45,6 +46,48 @@ export function verifyLocalUrlSignature(bucket: string, key: string, signature:
|
||||
}
|
||||
}
|
||||
|
||||
export const DEFAULT_LOCAL_GET_EXPIRY_SECONDS = 12 * 3600
|
||||
|
||||
export function signLocalGetUrl(
|
||||
bucket: string,
|
||||
key: string,
|
||||
expiresInSeconds: number = DEFAULT_LOCAL_GET_EXPIRY_SECONDS,
|
||||
): { signature: string; expires: number } {
|
||||
const expires = Math.floor(Date.now() / 1000) + expiresInSeconds
|
||||
const secret = process.env.BETTER_AUTH_SECRET || 'shumai-local-storage-secret'
|
||||
const hmac = crypto.createHmac('sha256', secret)
|
||||
hmac.update(`GET:${bucket}/${key}:${expires}`)
|
||||
const signature = hmac.digest('hex')
|
||||
return { signature, expires }
|
||||
}
|
||||
|
||||
export function verifyLocalGetUrlSignature(
|
||||
bucket: string,
|
||||
key: string,
|
||||
signature: string | undefined | null,
|
||||
expiresStr: string | undefined | null,
|
||||
): boolean {
|
||||
if (!signature || !expiresStr) return false
|
||||
const expires = parseInt(expiresStr, 10)
|
||||
if (isNaN(expires)) return false
|
||||
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
if (now > expires) return false
|
||||
|
||||
const secret = process.env.BETTER_AUTH_SECRET || 'shumai-local-storage-secret'
|
||||
const hmac = crypto.createHmac('sha256', secret)
|
||||
hmac.update(`GET:${bucket}/${key}:${expires}`)
|
||||
const expected = hmac.digest('hex')
|
||||
try {
|
||||
const sigBuf = Buffer.from(signature, 'hex')
|
||||
const expBuf = Buffer.from(expected, 'hex')
|
||||
if (sigBuf.length !== expBuf.length) return false
|
||||
return crypto.timingSafeEqual(sigBuf, expBuf)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a Content-Disposition header value for downloads.
|
||||
* - Falls back to plain `attachment` when no filename is provided.
|
||||
@@ -82,6 +125,241 @@ export interface S3Object {
|
||||
contentType: string
|
||||
}
|
||||
|
||||
export interface S3StreamObject {
|
||||
stream: ReadableStream<Uint8Array>
|
||||
contentLength?: number
|
||||
contentType?: string
|
||||
}
|
||||
|
||||
export interface UploadStreamParams {
|
||||
bucket: string
|
||||
key: string
|
||||
stream: ReadableStream<Uint8Array>
|
||||
contentLength?: number
|
||||
contentType?: string
|
||||
storageClass?: string
|
||||
}
|
||||
|
||||
const SINGLE_PUT_MAX_BYTES = 64 * 1024 * 1024 // 64 MB
|
||||
const MULTIPART_PART_SIZE_BYTES = 16 * 1024 * 1024 // 16 MB
|
||||
|
||||
export function bodyToWebReadableStream(body: unknown): ReadableStream<Uint8Array> {
|
||||
if (!body) {
|
||||
throw new Error('NoSuchKey: The specified key returned an empty body.')
|
||||
}
|
||||
if (
|
||||
typeof body === 'object' &&
|
||||
'transformToWebStream' in body &&
|
||||
typeof (body as { transformToWebStream: unknown }).transformToWebStream === 'function'
|
||||
) {
|
||||
return (
|
||||
body as { transformToWebStream: () => ReadableStream<Uint8Array> }
|
||||
).transformToWebStream()
|
||||
}
|
||||
if (typeof body === 'object' && 'getReader' in body) {
|
||||
return body as ReadableStream<Uint8Array>
|
||||
}
|
||||
// Convert Node.js Readable or AsyncIterable to Web ReadableStream
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const nodeReadable = 'pipe' in (body as any) ? (body as Readable) : Readable.from(body as any)
|
||||
return Readable.toWeb(nodeReadable) as unknown as ReadableStream<Uint8Array>
|
||||
}
|
||||
|
||||
export function isS3NotFoundError(err: unknown): boolean {
|
||||
if (!err || typeof err !== 'object') return false
|
||||
const r = err as Record<string, unknown>
|
||||
const name = typeof r.name === 'string' ? r.name : ''
|
||||
const code = typeof r.code === 'string' ? r.code : ''
|
||||
const msg = typeof r.message === 'string' ? r.message : ''
|
||||
const metadata = r.$metadata as { httpStatusCode?: number } | undefined
|
||||
return (
|
||||
name === 'NotFound' ||
|
||||
name === 'NoSuchKey' ||
|
||||
code === 'NotFound' ||
|
||||
code === 'NoSuchKey' ||
|
||||
code === 'ENOENT' ||
|
||||
metadata?.httpStatusCode === 404 ||
|
||||
msg.includes('NoSuchKey') ||
|
||||
msg.includes('NotFound')
|
||||
)
|
||||
}
|
||||
|
||||
export async function writeStreamToLocalFile(
|
||||
targetPath: string,
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
): Promise<void> {
|
||||
const dir = path.dirname(targetPath)
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true })
|
||||
}
|
||||
const tempPath = `${targetPath}.tmp-${ulid()}`
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const nodeStream = Readable.fromWeb(stream as any)
|
||||
await pipeline(nodeStream, fs.createWriteStream(tempPath))
|
||||
await fs.promises.rename(tempPath, targetPath)
|
||||
} catch (err) {
|
||||
await fs.promises.unlink(tempPath).catch(() => {})
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadStreamToS3Client(
|
||||
client: S3Client,
|
||||
params: UploadStreamParams,
|
||||
partSizeBytes: number = MULTIPART_PART_SIZE_BYTES,
|
||||
singlePutMaxBytes: number = SINGLE_PUT_MAX_BYTES,
|
||||
): Promise<void> {
|
||||
const { bucket, key, stream, contentLength, contentType, storageClass } = params
|
||||
// AWS SDK v3 StorageClass is an open string union at runtime; cast safely for arbitrary S3-compatible classes
|
||||
const s3StorageClass = storageClass ? (storageClass as StorageClass) : undefined
|
||||
|
||||
const MAX_S3_PARTS = 10000
|
||||
const effectivePartSize =
|
||||
contentLength !== undefined && contentLength > MAX_S3_PARTS * partSizeBytes
|
||||
? Math.max(partSizeBytes, Math.ceil(contentLength / 9900))
|
||||
: partSizeBytes
|
||||
|
||||
if (contentLength !== undefined && contentLength <= singlePutMaxBytes) {
|
||||
if (contentLength === 0) {
|
||||
await stream.cancel().catch(() => {})
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: Buffer.alloc(0),
|
||||
ContentLength: 0,
|
||||
ContentType: contentType,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
StorageClass: s3StorageClass,
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
// Convert Web ReadableStream to Node stream without buffering into memory
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const nodeStream = Readable.fromWeb(stream as any)
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
Body: nodeStream as any,
|
||||
ContentLength: contentLength,
|
||||
ContentType: contentType,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
StorageClass: s3StorageClass,
|
||||
}),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// Multipart streaming upload for > 64 MB or unknown contentLength
|
||||
const createRes = await client.send(
|
||||
new CreateMultipartUploadCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
ContentType: contentType,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
StorageClass: s3StorageClass,
|
||||
}),
|
||||
)
|
||||
const uploadId = createRes.UploadId
|
||||
if (!uploadId) {
|
||||
throw new Error(`Failed to initiate multipart upload for ${bucket}/${key}`)
|
||||
}
|
||||
|
||||
const reader = stream.getReader()
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const completedParts: Array<{ ETag?: string; PartNumber: number }> = []
|
||||
let partNumber = 1
|
||||
const pendingChunks: Uint8Array[] = []
|
||||
let pendingLength = 0
|
||||
|
||||
const flushPart = async (sizeToFlush: number) => {
|
||||
const partBuffer = Buffer.allocUnsafe(sizeToFlush)
|
||||
let offset = 0
|
||||
while (offset < sizeToFlush && pendingChunks.length > 0) {
|
||||
const chunk = pendingChunks[0]
|
||||
const remaining = sizeToFlush - offset
|
||||
if (chunk.byteLength <= remaining) {
|
||||
partBuffer.set(chunk, offset)
|
||||
offset += chunk.byteLength
|
||||
pendingLength -= chunk.byteLength
|
||||
pendingChunks.shift()
|
||||
} else {
|
||||
partBuffer.set(chunk.subarray(0, remaining), offset)
|
||||
pendingChunks[0] = chunk.subarray(remaining)
|
||||
offset += remaining
|
||||
pendingLength -= remaining
|
||||
}
|
||||
}
|
||||
const currentPartNumber = partNumber++
|
||||
const partRes = await client.send(
|
||||
new UploadPartCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
PartNumber: currentPartNumber,
|
||||
Body: partBuffer,
|
||||
ContentLength: partBuffer.byteLength,
|
||||
}),
|
||||
)
|
||||
completedParts.push({
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
ETag: partRes.ETag,
|
||||
PartNumber: currentPartNumber,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) {
|
||||
break
|
||||
}
|
||||
if (value && value.byteLength > 0) {
|
||||
pendingChunks.push(value)
|
||||
pendingLength += value.byteLength
|
||||
while (pendingLength >= effectivePartSize) {
|
||||
await flushPart(effectivePartSize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingLength > 0 || completedParts.length === 0) {
|
||||
await flushPart(pendingLength)
|
||||
}
|
||||
|
||||
await client.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
MultipartUpload: {
|
||||
Parts: completedParts,
|
||||
},
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
}),
|
||||
)
|
||||
} catch (err) {
|
||||
await reader.cancel().catch(() => {})
|
||||
await client
|
||||
.send(
|
||||
new AbortMultipartUploadCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
UploadId: uploadId,
|
||||
}),
|
||||
)
|
||||
.catch(() => {})
|
||||
throw err
|
||||
} finally {
|
||||
reader.releaseLock()
|
||||
}
|
||||
}
|
||||
|
||||
export interface S3Service {
|
||||
getObjectSize: (bucket: string, key: string) => Promise<number>
|
||||
putObject: (
|
||||
@@ -92,6 +370,8 @@ export interface S3Service {
|
||||
contentType?: string,
|
||||
) => Promise<void>
|
||||
getObject: (bucket: string, key: string) => Promise<S3Object>
|
||||
getObjectStream: (bucket: string, key: string) => Promise<S3StreamObject>
|
||||
uploadStream: (params: UploadStreamParams) => Promise<void>
|
||||
copyObject: (
|
||||
sourceBucket: string,
|
||||
sourceKey: string,
|
||||
@@ -112,7 +392,7 @@ export interface S3Service {
|
||||
download?: boolean,
|
||||
filename?: string,
|
||||
internal?: boolean,
|
||||
/** URL lifetime; defaults to PRESIGNED_URL_EXPIRES_IN hours. Ignored by local storage. */
|
||||
/** URL lifetime; defaults to PRESIGNED_UPLOAD_URL_EXPIRES_IN (12h) for PUT, or PRESIGNED_URL_EXPIRES_IN (5h) for GET, or DEFAULT_LOCAL_GET_EXPIRY_SECONDS (12h) for local. */
|
||||
expiresInSeconds?: number,
|
||||
) => Promise<string>
|
||||
presignMultipart: (
|
||||
@@ -122,6 +402,7 @@ export interface S3Service {
|
||||
) => Promise<{ url: string }>
|
||||
abortMultipartUpload: (bucket: string, key: string, uploadId: string) => Promise<void>
|
||||
resolveInput: (bucket: string, key: string) => Promise<string>
|
||||
getStoragePath: (bucket: string, key: string) => string
|
||||
}
|
||||
|
||||
export class S3StorageService implements S3Service {
|
||||
@@ -225,6 +506,19 @@ export class S3StorageService implements S3Service {
|
||||
}
|
||||
}
|
||||
|
||||
async getObjectStream(bucket: string, key: string): Promise<S3StreamObject> {
|
||||
const res = await this.client.send(new GetObjectCommand({ Bucket: bucket, Key: key }))
|
||||
return {
|
||||
stream: bodyToWebReadableStream(res.Body),
|
||||
contentLength: res.ContentLength,
|
||||
contentType: res.ContentType || Bun.file(key).type || 'application/octet-stream',
|
||||
}
|
||||
}
|
||||
|
||||
async uploadStream(params: UploadStreamParams): Promise<void> {
|
||||
await uploadStreamToS3Client(this.client, params)
|
||||
}
|
||||
|
||||
async copyObject(
|
||||
sourceBucket: string,
|
||||
sourceKey: string,
|
||||
@@ -267,8 +561,11 @@ export class S3StorageService implements S3Service {
|
||||
try {
|
||||
await this.client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key }))
|
||||
return 1
|
||||
} catch {
|
||||
return 0
|
||||
} catch (err: unknown) {
|
||||
if (isS3NotFoundError(err)) {
|
||||
return 0
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,7 +599,10 @@ export class S3StorageService implements S3Service {
|
||||
eTag: res.ETag || '',
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
throw new Error(`NoSuchKey: The specified key does not exist.`, { cause: err })
|
||||
if (isS3NotFoundError(err)) {
|
||||
throw new Error(`NoSuchKey: The specified key does not exist.`, { cause: err })
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,8 +664,16 @@ export class S3StorageService implements S3Service {
|
||||
internal?: boolean,
|
||||
expiresInSeconds?: number,
|
||||
): Promise<string> {
|
||||
const expireHours = parseInt(process.env.PRESIGNED_URL_EXPIRES_IN || '5', 10)
|
||||
const lifetimeSeconds = expiresInSeconds ?? expireHours * 3600
|
||||
const defaultExpireHours =
|
||||
method === 'PUT'
|
||||
? parseInt(
|
||||
process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN ||
|
||||
process.env.PRESIGNED_URL_EXPIRES_IN ||
|
||||
'12',
|
||||
10,
|
||||
)
|
||||
: parseInt(process.env.PRESIGNED_URL_EXPIRES_IN || '5', 10)
|
||||
const lifetimeSeconds = expiresInSeconds ?? defaultExpireHours * 3600
|
||||
// With an explicit (usually short) lifetime, only reuse a cached URL while it has at
|
||||
// least half of its lifetime left, so callers never receive a nearly expired link.
|
||||
const cacheTtlMs = expiresInSeconds ? (lifetimeSeconds * 1000) / 2 : lifetimeSeconds * 1000
|
||||
@@ -416,7 +724,10 @@ export class S3StorageService implements S3Service {
|
||||
key: string,
|
||||
request: S3SignRequest,
|
||||
): Promise<{ url: string }> {
|
||||
const expireHours = parseInt(process.env.PRESIGNED_URL_EXPIRES_IN || '5', 10)
|
||||
const expireHours = parseInt(
|
||||
process.env.PRESIGNED_UPLOAD_URL_EXPIRES_IN || process.env.PRESIGNED_URL_EXPIRES_IN || '12',
|
||||
10,
|
||||
)
|
||||
const expiresInSeconds = expireHours * 3600
|
||||
|
||||
let url: string
|
||||
@@ -492,11 +803,16 @@ export class S3StorageService implements S3Service {
|
||||
async resolveInput(bucket: string, key: string): Promise<string> {
|
||||
return this.presign(bucket, key, 'GET', false, undefined, true)
|
||||
}
|
||||
|
||||
getStoragePath(bucket: string, key: string): string {
|
||||
return `s3://${bucket}/${key}`
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalStorageService implements S3Service {
|
||||
private basePath: string
|
||||
private endpoint: string
|
||||
private presignCache = new LruTtlCache<string, string>(50000)
|
||||
|
||||
constructor(endpoint: string, basePath: string = 'data') {
|
||||
this.endpoint = endpoint.replace(/\/$/, '')
|
||||
@@ -572,6 +888,29 @@ export class LocalStorageService implements S3Service {
|
||||
}
|
||||
}
|
||||
|
||||
async getObjectStream(bucket: string, key: string): Promise<S3StreamObject> {
|
||||
const filePath = this.getFilePath(bucket, key)
|
||||
try {
|
||||
const stats = await fs.promises.stat(filePath)
|
||||
const nodeStream = fs.createReadStream(filePath)
|
||||
return {
|
||||
stream: Readable.toWeb(nodeStream) as unknown as ReadableStream<Uint8Array>,
|
||||
contentLength: stats.size,
|
||||
contentType: Bun.file(filePath).type || 'application/octet-stream',
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof Error && (e as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
throw new Error(`NoSuchKey: The specified key does not exist.`, { cause: e })
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async uploadStream(params: UploadStreamParams): Promise<void> {
|
||||
const filePath = this.getFilePath(params.bucket, params.key)
|
||||
await writeStreamToLocalFile(filePath, params.stream)
|
||||
}
|
||||
|
||||
async copyObject(
|
||||
sourceBucket: string,
|
||||
sourceKey: string,
|
||||
@@ -714,6 +1053,8 @@ export class LocalStorageService implements S3Service {
|
||||
method: string,
|
||||
download?: boolean,
|
||||
filename?: string,
|
||||
_internal?: boolean,
|
||||
expiresInSeconds?: number,
|
||||
): Promise<string> {
|
||||
if (method !== 'GET' && method !== 'PUT') {
|
||||
throw new Error(`Invalid method: ${method}`)
|
||||
@@ -721,13 +1062,31 @@ export class LocalStorageService implements S3Service {
|
||||
if (method === 'PUT') {
|
||||
return `${this.endpoint}${signLocalUrl(bucket, key)}`
|
||||
}
|
||||
let url = `${this.endpoint}/files/${bucket}/${key}`
|
||||
|
||||
const lifetimeSeconds = expiresInSeconds ?? DEFAULT_LOCAL_GET_EXPIRY_SECONDS
|
||||
const cacheTtlMs = (lifetimeSeconds * 1000) / 2
|
||||
const cacheKey = `${bucket}/${key}?download=${download || false}&filename=${filename || ''}&expires=${expiresInSeconds ?? 'default'}`
|
||||
|
||||
if (method === 'GET' && !download) {
|
||||
const cached = this.presignCache.get(cacheKey)
|
||||
if (cached) {
|
||||
return cached
|
||||
}
|
||||
}
|
||||
|
||||
const { signature, expires } = signLocalGetUrl(bucket, key, lifetimeSeconds)
|
||||
let url = `${this.endpoint}/files/${bucket}/${key}?Signature=${signature}&Expires=${expires}`
|
||||
if (download) {
|
||||
url += '?download=1'
|
||||
url += '&download=1'
|
||||
if (filename) {
|
||||
url += `&filename=${encodeURIComponent(filename)}`
|
||||
}
|
||||
}
|
||||
|
||||
if (method === 'GET' && !download) {
|
||||
this.presignCache.set(cacheKey, url, cacheTtlMs)
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -758,6 +1117,10 @@ export class LocalStorageService implements S3Service {
|
||||
}
|
||||
return this.presign(bucket, key, 'GET')
|
||||
}
|
||||
|
||||
getStoragePath(bucket: string, key: string): string {
|
||||
return this.getFilePath(bucket, key)
|
||||
}
|
||||
}
|
||||
|
||||
export function getStorageBackend(): 's3' | 'local' {
|
||||
|
||||
@@ -57,7 +57,7 @@ export class EmbeddingService {
|
||||
const candidateAssets = await prisma.asset.findMany({
|
||||
where: {
|
||||
isDeleted: false,
|
||||
status: 'processed',
|
||||
status: { in: ['processed', 'archived'] },
|
||||
type: { in: ['file', 'version_stack'] },
|
||||
project: { teamId: { in: activeTeamIds } },
|
||||
},
|
||||
|
||||
@@ -106,5 +106,34 @@ describe('SqlQueryBuilder', () => {
|
||||
)
|
||||
expect(query.values).toEqual(['custom_field"test', 'hello'])
|
||||
})
|
||||
|
||||
it('isArchived eq true matches archived files and version stacks whose latest version is archived', () => {
|
||||
const builder = new SqlQueryBuilder()
|
||||
.from(Prisma.sql`assets a`)
|
||||
.addSearchConditions('AND', [{ field: 'isArchived', operator: 'eq', value: true }])
|
||||
|
||||
const query = builder.build()
|
||||
expect(query.text).toContain(
|
||||
`a."type" = 'file'::"AssetType" AND a."status" = 'archived'::"AssetStatus"`,
|
||||
)
|
||||
})
|
||||
|
||||
it('isArchived eq false matches non-archived files and version stacks', () => {
|
||||
const builder = new SqlQueryBuilder()
|
||||
.from(Prisma.sql`assets a`)
|
||||
.addSearchConditions('AND', [{ field: 'isArchived', operator: 'eq', value: false }])
|
||||
|
||||
const query = builder.build()
|
||||
expect(query.text).toContain(
|
||||
`a."type" = 'file'::"AssetType" AND a."status" != 'archived'::"AssetStatus"`,
|
||||
)
|
||||
})
|
||||
|
||||
it('throws an error for unsupported isArchived operators', () => {
|
||||
const builder = new SqlQueryBuilder().from(Prisma.sql`assets a`)
|
||||
expect(() =>
|
||||
builder.addSearchConditions('AND', [{ field: 'isArchived', operator: 'gt', value: true }]),
|
||||
).toThrow('Unsupported operator for isArchived field: gt')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -285,6 +285,42 @@ export class SqlQueryBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
if (field === 'isArchived' || field === 'is_archived') {
|
||||
if (operator !== 'eq' && operator !== 'neq') {
|
||||
throw new Error(`Unsupported operator for isArchived field: ${operator}`)
|
||||
}
|
||||
const rawBool = value === true || value === 'true' || value === 1 || value === '1'
|
||||
const wantArchived = operator === 'neq' ? !rawBool : rawBool
|
||||
if (wantArchived) {
|
||||
return Prisma.sql`(
|
||||
(a."type" = 'file'::"AssetType" AND a."status" = 'archived'::"AssetStatus")
|
||||
OR (
|
||||
a."type" = 'version_stack'::"AssetType"
|
||||
AND (
|
||||
SELECT v."status" FROM assets v
|
||||
WHERE v."parent_id" = a."id" AND v."is_deleted" = false
|
||||
ORDER BY v."sort_index" ASC LIMIT 1
|
||||
) = 'archived'::"AssetStatus"
|
||||
)
|
||||
)`
|
||||
}
|
||||
return Prisma.sql`(
|
||||
(a."type" = 'file'::"AssetType" AND a."status" != 'archived'::"AssetStatus")
|
||||
OR (
|
||||
a."type" = 'version_stack'::"AssetType"
|
||||
AND COALESCE(
|
||||
(
|
||||
SELECT v."status"::text FROM assets v
|
||||
WHERE v."parent_id" = a."id" AND v."is_deleted" = false
|
||||
ORDER BY v."sort_index" ASC LIMIT 1
|
||||
),
|
||||
''
|
||||
) != 'archived'
|
||||
)
|
||||
OR a."type" NOT IN ('file'::"AssetType", 'version_stack'::"AssetType")
|
||||
)`
|
||||
}
|
||||
|
||||
if (
|
||||
field === 'createdAt' ||
|
||||
field === 'updatedAt' ||
|
||||
|
||||
@@ -150,11 +150,11 @@ describe('ShareService', () => {
|
||||
expect(symlink).toBeDefined()
|
||||
|
||||
// Test password check
|
||||
await expect(shareService.verifyPublicAccess(symlink!.id, 'wrong')).rejects.toThrow(
|
||||
'Invalid password',
|
||||
)
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(symlink!.id, 'secret-password'),
|
||||
shareService.verifyPublicAccess(shareLink.id, symlink!.id, 'wrong'),
|
||||
).rejects.toThrow('Invalid password')
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(shareLink.id, symlink!.id, 'secret-password'),
|
||||
).resolves.toBeDefined()
|
||||
|
||||
// Test expiration
|
||||
@@ -162,9 +162,9 @@ describe('ShareService', () => {
|
||||
where: { id: shareLink.id },
|
||||
data: { expireAt: new Date(Date.now() - 1000) },
|
||||
})
|
||||
await expect(shareService.verifyPublicAccess(symlink!.id, 'secret-password')).rejects.toThrow(
|
||||
'expired',
|
||||
)
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(shareLink.id, symlink!.id, 'secret-password'),
|
||||
).rejects.toThrow('expired')
|
||||
})
|
||||
|
||||
it('traces deep ancestors for public access', async () => {
|
||||
@@ -192,7 +192,7 @@ describe('ShareService', () => {
|
||||
},
|
||||
})
|
||||
|
||||
await expect(shareService.verifyPublicAccess(deepFile.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, deepFile.id)).resolves.toBeDefined()
|
||||
})
|
||||
|
||||
it('verifies public access via symlink to a folder', async () => {
|
||||
@@ -220,10 +220,11 @@ describe('ShareService', () => {
|
||||
await shareService.addAssetToShare(shareLink.id, { assetIds: [realFolder.id] })
|
||||
|
||||
// 3. Verify access to the file inside the real folder (which is reached via the symlinked folder)
|
||||
await expect(shareService.verifyPublicAccess(realFile.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, realFile.id)).resolves.toBeDefined()
|
||||
})
|
||||
|
||||
it('fails public access for non-shared assets', async () => {
|
||||
const shareLink = await shareService.createShareLink(projectId, { name: 'Empty Share' })
|
||||
const privateFile = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'private.txt',
|
||||
@@ -232,7 +233,7 @@ describe('ShareService', () => {
|
||||
projectId,
|
||||
},
|
||||
})
|
||||
await expect(shareService.verifyPublicAccess(privateFile.id)).rejects.toThrow(
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, privateFile.id)).rejects.toThrow(
|
||||
'Asset is not shared',
|
||||
)
|
||||
})
|
||||
@@ -373,8 +374,8 @@ describe('ShareService', () => {
|
||||
where: { targetId: file.id, parentId: shareLink.rootFolderId },
|
||||
})
|
||||
expect(symlink).toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(file.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(symlink!.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, file.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, symlink!.id)).resolves.toBeDefined()
|
||||
|
||||
// 1. Soft-delete the file (move to trash)
|
||||
await assetService.deleteAssets([file.id])
|
||||
@@ -393,8 +394,8 @@ describe('ShareService', () => {
|
||||
expect(afterDeleteShareRoot?.fileCount).toBe(0)
|
||||
|
||||
// Verify: public access is rejected
|
||||
await expect(shareService.verifyPublicAccess(file.id)).rejects.toThrow()
|
||||
await expect(shareService.verifyPublicAccess(symlink!.id)).rejects.toThrow()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, file.id)).rejects.toThrow()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, symlink!.id)).rejects.toThrow()
|
||||
|
||||
// Verify: project's recently-deleted view does NOT list symlinks
|
||||
const recentlyDeleted = await assetService.listChildren({
|
||||
@@ -423,8 +424,8 @@ describe('ShareService', () => {
|
||||
expect(afterRestoreShareRoot?.fileCount).toBe(1)
|
||||
|
||||
// Verify: public access works again
|
||||
await expect(shareService.verifyPublicAccess(file.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(symlink!.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, file.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, symlink!.id)).resolves.toBeDefined()
|
||||
})
|
||||
|
||||
it('hides soft-deleted folders from share link and restores them upon folder restoration', async () => {
|
||||
@@ -463,7 +464,7 @@ describe('ShareService', () => {
|
||||
})
|
||||
expect(initialChildren.data).toHaveLength(1)
|
||||
expect(initialChildren.data[0].name).toBe('Shared Folder')
|
||||
await expect(shareService.verifyPublicAccess(childFile.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, childFile.id)).resolves.toBeDefined()
|
||||
|
||||
// 1. Soft-delete the folder
|
||||
await assetService.deleteAssets([folder.id])
|
||||
@@ -476,7 +477,7 @@ describe('ShareService', () => {
|
||||
expect(afterDeleteChildren.data).toHaveLength(0)
|
||||
|
||||
// Verify: access to child file inside folder is rejected
|
||||
await expect(shareService.verifyPublicAccess(childFile.id)).rejects.toThrow()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, childFile.id)).rejects.toThrow()
|
||||
|
||||
// 2. Restore the folder
|
||||
await assetService.restoreAssets([folder.id])
|
||||
@@ -490,6 +491,46 @@ describe('ShareService', () => {
|
||||
expect(afterRestoreChildren.data[0].name).toBe('Shared Folder')
|
||||
|
||||
// Verify: access to child file inside folder is restored
|
||||
await expect(shareService.verifyPublicAccess(childFile.id)).resolves.toBeDefined()
|
||||
await expect(shareService.verifyPublicAccess(shareLink.id, childFile.id)).resolves.toBeDefined()
|
||||
})
|
||||
|
||||
it('binds public access strictly to the requested shareId', async () => {
|
||||
const fileA = await prisma.asset.create({
|
||||
data: { name: 'fileA.txt', type: AssetType.file, status: 'processed', projectId },
|
||||
})
|
||||
const fileB = await prisma.asset.create({
|
||||
data: { name: 'fileB.txt', type: AssetType.file, status: 'processed', projectId },
|
||||
})
|
||||
|
||||
const shareOpen = await shareService.createShareLink(projectId, { name: 'Open Share' })
|
||||
const shareProtected = await shareService.createShareLink(projectId, {
|
||||
name: 'Protected Share',
|
||||
password: 'password123',
|
||||
})
|
||||
|
||||
await shareService.addAssetToShare(shareOpen.id, { assetIds: [fileA.id, fileB.id] })
|
||||
await shareService.addAssetToShare(shareProtected.id, { assetIds: [fileA.id] })
|
||||
|
||||
// fileA in protected share requires password
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(shareProtected.id, fileA.id, 'wrong'),
|
||||
).rejects.toThrow('Invalid password')
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(shareProtected.id, fileA.id, 'password123'),
|
||||
).resolves.toBeDefined()
|
||||
|
||||
// fileB is only in shareOpen, accessing it via shareProtected must fail even with password
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(shareProtected.id, fileB.id, 'password123'),
|
||||
).rejects.toThrow('Asset is not shared')
|
||||
|
||||
// Disabled share must reject access even if file sits in another active share
|
||||
await prisma.shareLink.update({
|
||||
where: { id: shareProtected.id },
|
||||
data: { isDisabled: true },
|
||||
})
|
||||
await expect(
|
||||
shareService.verifyPublicAccess(shareProtected.id, fileA.id, 'password123'),
|
||||
).rejects.toThrow('disabled')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -249,57 +249,16 @@ export class ShareService {
|
||||
}
|
||||
|
||||
async verifyPublicAccess(
|
||||
shareLinkId: string,
|
||||
assetId: string,
|
||||
providedPassword?: string,
|
||||
): Promise<Prisma.ShareLinkGetPayload<true>> {
|
||||
// 1. Trace ancestors of the current asset in the real tree
|
||||
// 2. Find if any ancestor (including itself) is the target of a symlink that belongs to a share
|
||||
// 3. Or if the asset itself is a descendant of a 'share' type folder (direct share link root or subfolder within)
|
||||
|
||||
const rows = await prisma.$queryRaw<{ shareRootId: string }[]>`
|
||||
WITH RECURSIVE
|
||||
-- Trace ancestors in the real asset tree
|
||||
real_ancestors AS (
|
||||
SELECT id, parent_id, type::text FROM assets WHERE id = ${assetId} AND is_deleted = false
|
||||
UNION ALL
|
||||
SELECT a.id, a.parent_id, a.type::text FROM assets a
|
||||
INNER JOIN real_ancestors ra ON ra.parent_id = a.id
|
||||
WHERE a.is_deleted = false
|
||||
),
|
||||
-- Find all symlinks pointing to any of these real ancestors
|
||||
found_symlinks AS (
|
||||
SELECT s.id, s.parent_id, s.type::text FROM assets s
|
||||
INNER JOIN real_ancestors ra ON s.target_id = ra.id
|
||||
WHERE s.type = 'symlink' AND s.is_deleted = false
|
||||
),
|
||||
-- Combine: start with either real ancestors that are 'share' type, or symlinks that lead to 'share' type
|
||||
search_starts AS (
|
||||
SELECT id, parent_id, type FROM real_ancestors WHERE type = 'share'
|
||||
UNION
|
||||
SELECT id, parent_id, type FROM found_symlinks
|
||||
),
|
||||
-- Trace UP from these search starts to find the ultimate 'share' root
|
||||
share_trace AS (
|
||||
SELECT id, parent_id, type FROM search_starts
|
||||
UNION ALL
|
||||
SELECT a.id, a.parent_id, a.type::text FROM assets a
|
||||
INNER JOIN share_trace st ON st.parent_id = a.id
|
||||
WHERE a.is_deleted = false
|
||||
)
|
||||
SELECT id as "shareRootId" FROM share_trace WHERE type = 'share' LIMIT 1;
|
||||
`
|
||||
|
||||
if (rows.length === 0) {
|
||||
throw new ShareLinkNotFoundError('Asset is not shared')
|
||||
}
|
||||
|
||||
const shareRootId = rows[0].shareRootId
|
||||
const shareLink = await prisma.shareLink.findUnique({
|
||||
where: { rootFolderId: shareRootId },
|
||||
where: { id: shareLinkId },
|
||||
})
|
||||
|
||||
if (!shareLink) {
|
||||
throw new ShareLinkNotFoundError('Share link not found for this asset')
|
||||
throw new ShareLinkNotFoundError('Share link not found')
|
||||
}
|
||||
|
||||
if (shareLink.isDisabled) {
|
||||
@@ -314,6 +273,43 @@ export class ShareService {
|
||||
throw new ShareLinkPasswordInvalidError('Invalid password for share link')
|
||||
}
|
||||
|
||||
const rows = await prisma.$queryRaw<{ id: string }[]>`
|
||||
WITH RECURSIVE
|
||||
-- Trace ancestors in the real asset tree
|
||||
real_ancestors AS (
|
||||
SELECT id, parent_id, type::text, target_id FROM assets WHERE id = ${assetId} AND is_deleted = false
|
||||
UNION ALL
|
||||
SELECT a.id, a.parent_id, a.type::text, a.target_id FROM assets a
|
||||
INNER JOIN real_ancestors ra ON ra.parent_id = a.id
|
||||
WHERE a.is_deleted = false
|
||||
),
|
||||
-- Find all symlinks pointing to any of these real ancestors
|
||||
found_symlinks AS (
|
||||
SELECT s.id, s.parent_id, s.type::text FROM assets s
|
||||
INNER JOIN real_ancestors ra ON s.target_id = ra.id
|
||||
WHERE s.type = 'symlink' AND s.is_deleted = false
|
||||
),
|
||||
-- Combine: start with either real ancestors, or symlinks that lead to them
|
||||
search_starts AS (
|
||||
SELECT id, parent_id, type FROM real_ancestors
|
||||
UNION
|
||||
SELECT id, parent_id, type FROM found_symlinks
|
||||
),
|
||||
-- Trace UP from these search starts to find the ultimate 'share' root
|
||||
share_trace AS (
|
||||
SELECT id, parent_id, type FROM search_starts
|
||||
UNION ALL
|
||||
SELECT a.id, a.parent_id, a.type::text FROM assets a
|
||||
INNER JOIN share_trace st ON st.parent_id = a.id
|
||||
WHERE a.is_deleted = false
|
||||
)
|
||||
SELECT id FROM share_trace WHERE id = ${shareLink.rootFolderId} LIMIT 1;
|
||||
`
|
||||
|
||||
if (rows.length === 0) {
|
||||
throw new ShareLinkNotFoundError('Asset is not shared')
|
||||
}
|
||||
|
||||
return shareLink
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import * as path from 'path'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { teamService } from '@shumai/core/src/team/team'
|
||||
@@ -388,6 +389,13 @@ describe('TeamService', () => {
|
||||
username: undefined,
|
||||
},
|
||||
appearance: { hideAgent: false },
|
||||
archiveStorage: { enabled: false },
|
||||
autoArchive: {
|
||||
enabled: false,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
})
|
||||
|
||||
await teamService.updateSettings(team.id, 'theme', 'dark')
|
||||
@@ -417,6 +425,13 @@ describe('TeamService', () => {
|
||||
username: undefined,
|
||||
},
|
||||
appearance: { hideAgent: false },
|
||||
archiveStorage: { enabled: false },
|
||||
autoArchive: {
|
||||
enabled: false,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
})
|
||||
|
||||
await teamService.updateSettings(team.id, 'transcode.videoStrategy', 'multi')
|
||||
@@ -456,6 +471,13 @@ describe('TeamService', () => {
|
||||
},
|
||||
appearance: { hideAgent: true },
|
||||
semanticSearchEnabled: false,
|
||||
archiveStorage: { enabled: false },
|
||||
autoArchive: {
|
||||
enabled: false,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -633,4 +655,66 @@ describe('TeamService', () => {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects enabling auto-archive when archive storage is not configured', async () => {
|
||||
delete process.env.ARCHIVE_STORAGE_BACKEND
|
||||
const owner = await prisma.user.create({
|
||||
data: { name: 'Owner AA', email: 'owner-aa@example.com', password: 'pw' },
|
||||
})
|
||||
const team = await teamService.createTeam(owner, { name: 'AA Team' })
|
||||
|
||||
await expect(
|
||||
teamService.updateSettings(team.id, 'autoArchive', {
|
||||
enabled: true,
|
||||
method: 'not_updated',
|
||||
days: 30,
|
||||
postRestoreGraceDays: 30,
|
||||
}),
|
||||
).rejects.toThrow('Cannot enable auto archive when archive storage is not configured')
|
||||
|
||||
await expect(teamService.updateSettings(team.id, 'autoArchive.enabled', true)).rejects.toThrow(
|
||||
'Cannot enable auto archive when archive storage is not configured',
|
||||
)
|
||||
})
|
||||
|
||||
it('allows updating auto-archive settings when archive storage is configured', async () => {
|
||||
const saved = { ...process.env }
|
||||
process.env.ARCHIVE_STORAGE_BACKEND = 'local'
|
||||
process.env.ARCHIVE_LOCAL_ROOT = '/tmp/shumai-archive-test-team'
|
||||
|
||||
try {
|
||||
const owner = await prisma.user.create({
|
||||
data: { name: 'Owner AA 2', email: 'owner-aa2@example.com', password: 'pw' },
|
||||
})
|
||||
const team = await teamService.createTeam(owner, { name: 'AA Team 2' })
|
||||
|
||||
await teamService.updateSettings(team.id, 'autoArchive', {
|
||||
enabled: true,
|
||||
method: 'after_created',
|
||||
days: 45,
|
||||
postRestoreGraceDays: 15,
|
||||
})
|
||||
|
||||
const settings = await teamService.getSettings(team.id)
|
||||
expect(settings.autoArchive).toEqual({
|
||||
enabled: true,
|
||||
method: 'after_created',
|
||||
days: 45,
|
||||
postRestoreGraceDays: 15,
|
||||
})
|
||||
expect(settings.archiveStorage).toEqual({
|
||||
enabled: true,
|
||||
backend: 'local',
|
||||
localRoot: path.resolve('/tmp/shumai-archive-test-team'),
|
||||
})
|
||||
|
||||
// Non-admin call should mask backend/root details
|
||||
const readerSettings = await teamService.getSettings(team.id, { isAdmin: false })
|
||||
expect(readerSettings.archiveStorage).toEqual({
|
||||
enabled: true,
|
||||
})
|
||||
} finally {
|
||||
process.env = saved
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -10,6 +10,10 @@ import { AgentService, agentService } from '@shumai/core/src/agent/agent'
|
||||
import { sandboxService } from '@shumai/core/src/sandbox/sandbox-service'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
import {
|
||||
getArchiveStorageConfig,
|
||||
isArchiveStorageEnabled,
|
||||
} from '@shumai/core/src/s3/archive-storage'
|
||||
import {
|
||||
ServiceCreateTeamRequest,
|
||||
ServiceGetUserTeamsRequest,
|
||||
@@ -23,6 +27,9 @@ import {
|
||||
SandboxSettings,
|
||||
UpdateSandboxSettingsRequest,
|
||||
TeamSettingsResponse,
|
||||
AutoArchiveMethod,
|
||||
AutoArchiveSettings,
|
||||
DEFAULT_AUTO_ARCHIVE_SETTINGS,
|
||||
} from '@shumai/dtos'
|
||||
|
||||
export class TeamService {
|
||||
@@ -300,7 +307,10 @@ export class TeamService {
|
||||
})
|
||||
}
|
||||
|
||||
async getSettings(teamId: string): Promise<TeamSettingsResponse> {
|
||||
async getSettings(
|
||||
teamId: string,
|
||||
options?: { isAdmin?: boolean },
|
||||
): Promise<TeamSettingsResponse> {
|
||||
const team = await prisma.team.findUnique({
|
||||
where: { id: teamId },
|
||||
})
|
||||
@@ -355,6 +365,33 @@ export class TeamService {
|
||||
settings.turntable = await turntableService.getSettings(teamId)
|
||||
settings.gotenberg = await gotenbergService.getSettings(teamId)
|
||||
|
||||
const archiveConfig = getArchiveStorageConfig()
|
||||
if (!archiveConfig.enabled) {
|
||||
settings.archiveStorage = { enabled: false }
|
||||
} else if (options?.isAdmin === false) {
|
||||
settings.archiveStorage = { enabled: true }
|
||||
} else if (archiveConfig.backend === 'local') {
|
||||
settings.archiveStorage = {
|
||||
enabled: true,
|
||||
backend: 'local',
|
||||
localRoot: archiveConfig.localRoot,
|
||||
}
|
||||
} else {
|
||||
settings.archiveStorage = {
|
||||
enabled: true,
|
||||
backend: 's3',
|
||||
endpoint: archiveConfig.endpoint,
|
||||
bucket: archiveConfig.bucket,
|
||||
}
|
||||
}
|
||||
|
||||
settings.autoArchive = {
|
||||
enabled: settings.autoArchive?.enabled ?? false,
|
||||
method: settings.autoArchive?.method ?? 'not_updated',
|
||||
days: settings.autoArchive?.days ?? 30,
|
||||
postRestoreGraceDays: settings.autoArchive?.postRestoreGraceDays ?? 30,
|
||||
}
|
||||
|
||||
return settings
|
||||
}
|
||||
|
||||
@@ -421,6 +458,49 @@ export class TeamService {
|
||||
}
|
||||
settings.appearance.hideAgent = Boolean(value)
|
||||
delete settings['appearance.hideAgent']
|
||||
} else if (key === 'autoArchive') {
|
||||
const autoArchiveVal = value as AutoArchiveSettings
|
||||
if (autoArchiveVal.enabled && !isArchiveStorageEnabled()) {
|
||||
throw new HTTPException(400, {
|
||||
message: 'Cannot enable auto archive when archive storage is not configured',
|
||||
})
|
||||
}
|
||||
settings.autoArchive = {
|
||||
enabled: Boolean(autoArchiveVal.enabled),
|
||||
method: autoArchiveVal.method || 'not_updated',
|
||||
days: Number(autoArchiveVal.days) || 30,
|
||||
postRestoreGraceDays: Number(autoArchiveVal.postRestoreGraceDays) || 30,
|
||||
}
|
||||
} else if (key === 'autoArchive.enabled') {
|
||||
const enabled = Boolean(value)
|
||||
if (enabled && !isArchiveStorageEnabled()) {
|
||||
throw new HTTPException(400, {
|
||||
message: 'Cannot enable auto archive when archive storage is not configured',
|
||||
})
|
||||
}
|
||||
if (!settings.autoArchive) {
|
||||
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
|
||||
}
|
||||
settings.autoArchive.enabled = enabled
|
||||
delete settings['autoArchive.enabled']
|
||||
} else if (key === 'autoArchive.method') {
|
||||
if (!settings.autoArchive) {
|
||||
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
|
||||
}
|
||||
settings.autoArchive.method = value as AutoArchiveMethod
|
||||
delete settings['autoArchive.method']
|
||||
} else if (key === 'autoArchive.days') {
|
||||
if (!settings.autoArchive) {
|
||||
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
|
||||
}
|
||||
settings.autoArchive.days = Number(value)
|
||||
delete settings['autoArchive.days']
|
||||
} else if (key === 'autoArchive.postRestoreGraceDays') {
|
||||
if (!settings.autoArchive) {
|
||||
settings.autoArchive = { ...DEFAULT_AUTO_ARCHIVE_SETTINGS }
|
||||
}
|
||||
settings.autoArchive.postRestoreGraceDays = Number(value)
|
||||
delete settings['autoArchive.postRestoreGraceDays']
|
||||
} else {
|
||||
settings[key] = value
|
||||
}
|
||||
|
||||
@@ -170,6 +170,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -228,6 +229,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -268,6 +270,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -303,6 +306,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -336,6 +340,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -366,6 +371,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -399,6 +405,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -429,6 +436,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -484,6 +492,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -512,6 +521,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: htmlTask.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -547,6 +557,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -584,6 +595,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -616,6 +628,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -649,6 +662,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -752,6 +766,7 @@ describe('UploadService', () => {
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: stack.id } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
@@ -797,6 +812,7 @@ describe('UploadService', () => {
|
||||
expect(result.data).toHaveLength(1)
|
||||
expect(result.data[0].id).toBe(task.id)
|
||||
expect(result.data[0].name).toBe('test-upload-task')
|
||||
expect(result.data[0].status).toBe(TaskStatus.pending)
|
||||
})
|
||||
|
||||
it('should correct the mediaType for .wma files that browser incorrectly reports as video', async () => {
|
||||
@@ -1119,5 +1135,325 @@ describe('UploadService', () => {
|
||||
})
|
||||
expect(deletedAsset).toBeNull()
|
||||
})
|
||||
|
||||
it('should not increment task count if error arrives for an already finished or deleted asset', async () => {
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
creatorId: userId,
|
||||
type: 'upload',
|
||||
name: 'task-late-err',
|
||||
total: 2,
|
||||
uploaded: 1,
|
||||
status: TaskStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'files/test/already-uploaded.mp4' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'already-uploaded.mp4',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
storageKeyId: storageKey.id,
|
||||
taskId: task.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
|
||||
await uploadService.confirmFileUpload(userId, task.id, {
|
||||
fileId: asset.id,
|
||||
errorMessage: 'Late network error',
|
||||
})
|
||||
|
||||
const taskAfter = await prisma.task.findUniqueOrThrow({ where: { id: task.id } })
|
||||
expect(taskAfter.uploaded).toBe(1)
|
||||
})
|
||||
|
||||
it('should reject confirmation if asset does not belong to specified task', async () => {
|
||||
const task1 = await prisma.task.create({
|
||||
data: { creatorId: userId, type: 'upload', total: 1, status: TaskStatus.uploading },
|
||||
})
|
||||
const task2 = await prisma.task.create({
|
||||
data: { creatorId: userId, type: 'upload', total: 1, status: TaskStatus.uploading },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'files/test/task-mismatch.mp4' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'task-mismatch.mp4',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
storageKeyId: storageKey.id,
|
||||
taskId: task1.id,
|
||||
status: AssetStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(
|
||||
uploadService.confirmFileUpload(userId, task2.id, { fileId: asset.id }),
|
||||
).rejects.toThrow('Asset does not belong to specified task')
|
||||
})
|
||||
})
|
||||
|
||||
describe('cancelUploadTask', () => {
|
||||
it('cancels uploading assets, leaves uploaded assets and folders intact, and marks task failed', async () => {
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
creatorId: userId,
|
||||
type: 'upload',
|
||||
name: 'multi-file-task',
|
||||
total: 3,
|
||||
uploaded: 1,
|
||||
status: TaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
// 1. A folder created by the task
|
||||
const folder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'subfolder',
|
||||
type: AssetType.folder,
|
||||
projectId,
|
||||
parentId,
|
||||
taskId: task.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. An asset already finished uploading
|
||||
const finishedKey = await prisma.storageKey.create({ data: { key: 'files/test/done.mp4' } })
|
||||
const finishedAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'done.mp4',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId: folder.id,
|
||||
storageKeyId: finishedKey.id,
|
||||
taskId: task.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
|
||||
// 3. An incomplete multipart asset
|
||||
const multipartKey = await prisma.storageKey.create({
|
||||
data: { key: 'files/test/part.mp4' },
|
||||
})
|
||||
const multipartAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'part.mp4',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId: folder.id,
|
||||
storageKeyId: multipartKey.id,
|
||||
taskId: task.id,
|
||||
uploadId: 'upload-multipart-cancel',
|
||||
status: AssetStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
// 4. An incomplete single PUT asset
|
||||
const singleKey = await prisma.storageKey.create({
|
||||
data: { key: 'files/test/single.mp4' },
|
||||
})
|
||||
const singleAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'single.mp4',
|
||||
type: AssetType.file,
|
||||
projectId,
|
||||
parentId: folder.id,
|
||||
storageKeyId: singleKey.id,
|
||||
taskId: task.id,
|
||||
status: AssetStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
const res = await uploadService.cancelUploadTask(teamId, userId, task.id)
|
||||
expect(res).toEqual({ success: true })
|
||||
|
||||
// Incomplete assets deleted
|
||||
expect(await prisma.asset.findUnique({ where: { id: multipartAsset.id } })).toBeNull()
|
||||
expect(await prisma.storageKey.findUnique({ where: { id: multipartKey.id } })).toBeNull()
|
||||
expect(await prisma.asset.findUnique({ where: { id: singleAsset.id } })).toBeNull()
|
||||
expect(await prisma.storageKey.findUnique({ where: { id: singleKey.id } })).toBeNull()
|
||||
|
||||
// S3 cleanup called
|
||||
expect(s3Service.abortMultipartUpload).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'files/test/part.mp4',
|
||||
'upload-multipart-cancel',
|
||||
)
|
||||
expect(s3Service.deleteObject).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'files/test/single.mp4',
|
||||
)
|
||||
|
||||
// Completed asset and folder remain intact
|
||||
expect(await prisma.asset.findUnique({ where: { id: finishedAsset.id } })).not.toBeNull()
|
||||
expect(await prisma.storageKey.findUnique({ where: { id: finishedKey.id } })).not.toBeNull()
|
||||
expect(await prisma.asset.findUnique({ where: { id: folder.id } })).not.toBeNull()
|
||||
|
||||
// Task status updated to failed
|
||||
const updatedTask = await prisma.task.findUnique({ where: { id: task.id } })
|
||||
expect(updatedTask?.status).toBe(TaskStatus.failed)
|
||||
expect(updatedTask?.uploaded).toBe(1)
|
||||
})
|
||||
|
||||
it('rejects cancellation if the caller is not the task creator', async () => {
|
||||
const otherUser = await prisma.user.create({
|
||||
data: {
|
||||
email: 'other@example.com',
|
||||
name: 'Other User',
|
||||
},
|
||||
})
|
||||
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
creatorId: otherUser.id,
|
||||
type: 'upload',
|
||||
name: 'other-task',
|
||||
total: 1,
|
||||
status: TaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(uploadService.cancelUploadTask(teamId, userId, task.id)).rejects.toThrow(
|
||||
'Unauthorized to cancel this upload task',
|
||||
)
|
||||
})
|
||||
|
||||
it('throws when task does not exist', async () => {
|
||||
await expect(
|
||||
uploadService.cancelUploadTask(teamId, userId, 'non-existent-task'),
|
||||
).rejects.toThrow('Upload task not found')
|
||||
})
|
||||
|
||||
it('throws when task assets belong to a different team', async () => {
|
||||
const otherTeam = await prisma.team.create({
|
||||
data: { name: 'Other Team' },
|
||||
})
|
||||
const otherProject = await prisma.project.create({
|
||||
data: { name: 'Other Project', teamId: otherTeam.id },
|
||||
})
|
||||
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
creatorId: userId,
|
||||
type: 'upload',
|
||||
name: 'wrong-team-task',
|
||||
total: 1,
|
||||
status: TaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'wrong-team-file.mp4',
|
||||
type: AssetType.file,
|
||||
projectId: otherProject.id,
|
||||
taskId: task.id,
|
||||
status: AssetStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(uploadService.cancelUploadTask(teamId, userId, task.id)).rejects.toThrow(
|
||||
'Task does not belong to specified team',
|
||||
)
|
||||
})
|
||||
|
||||
it('throws when task is already completed and preserves completed status', async () => {
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
creatorId: userId,
|
||||
type: 'upload',
|
||||
name: 'completed-task',
|
||||
total: 1,
|
||||
uploaded: 1,
|
||||
status: TaskStatus.completed,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(uploadService.cancelUploadTask(teamId, userId, task.id)).rejects.toThrow(
|
||||
'Cannot cancel a completed upload task',
|
||||
)
|
||||
|
||||
const refreshed = await prisma.task.findUnique({ where: { id: task.id } })
|
||||
expect(refreshed?.status).toBe(TaskStatus.completed)
|
||||
})
|
||||
|
||||
it('preserves an asset and its storage if it was confirmed uploaded concurrently', async () => {
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
creatorId: userId,
|
||||
type: 'upload',
|
||||
name: 'race-task',
|
||||
total: 2,
|
||||
uploaded: 0,
|
||||
status: TaskStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
const confirmedAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'race-confirmed.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
create: { key: 'files/test/race-confirmed.mp4' },
|
||||
},
|
||||
},
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
const cancelledAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'race-cancelled.mp4',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
task: { connect: { id: task.id } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
create: { key: 'files/test/race-cancelled.mp4' },
|
||||
},
|
||||
},
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
// Simulate concurrent confirm: update status to uploaded
|
||||
await prisma.asset.update({
|
||||
where: { id: confirmedAsset.id },
|
||||
data: { status: AssetStatus.uploaded },
|
||||
})
|
||||
|
||||
const deleteObjectSpy = vi.spyOn(s3Service, 'deleteObject').mockResolvedValue(1)
|
||||
|
||||
await uploadService.cancelUploadTask(teamId, userId, task.id)
|
||||
|
||||
// Confirmed asset must still exist in DB
|
||||
const dbConfirmed = await prisma.asset.findUnique({ where: { id: confirmedAsset.id } })
|
||||
expect(dbConfirmed).not.toBeNull()
|
||||
expect(dbConfirmed?.status).toBe(AssetStatus.uploaded)
|
||||
|
||||
// Cancelled asset must be deleted
|
||||
const dbCancelled = await prisma.asset.findUnique({ where: { id: cancelledAsset.id } })
|
||||
expect(dbCancelled).toBeNull()
|
||||
|
||||
// S3 deleteObject was called for the cancelled asset, but NOT for the confirmed asset
|
||||
expect(deleteObjectSpy).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'files/test/race-cancelled.mp4',
|
||||
)
|
||||
expect(deleteObjectSpy).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'files/test/race-confirmed.mp4',
|
||||
)
|
||||
|
||||
deleteObjectSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,6 +5,7 @@ import { AssetStatus, AssetType, Prisma, TaskStatus, prisma } from '@shumai/db'
|
||||
import {
|
||||
AbortUploadRequest,
|
||||
AbortUploadResponse,
|
||||
CancelUploadTaskResponse,
|
||||
ConfirmFileUploadRequest,
|
||||
CreateUploadTaskRequest,
|
||||
CreateUploadTaskResponse,
|
||||
@@ -205,51 +206,37 @@ export class UploadService {
|
||||
},
|
||||
})
|
||||
if (!asset) throw new Error('Asset not found')
|
||||
if (asset.taskId !== taskId) {
|
||||
throw new Error('Asset does not belong to specified task')
|
||||
}
|
||||
if (!asset.project) throw new Error('Project not found for asset')
|
||||
const key = asset.storageKey?.key
|
||||
if (!key) throw new Error('Asset has no key')
|
||||
|
||||
if (req.errorMessage) {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
if (asset.uploadId && key) {
|
||||
try {
|
||||
await s3Service.abortMultipartUpload(bucket, key, asset.uploadId)
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
{ err, key, uploadId: asset.uploadId },
|
||||
'Failed to abort multipart upload on confirm error',
|
||||
)
|
||||
}
|
||||
} else if (key) {
|
||||
try {
|
||||
await s3Service.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
logger.warn({ err, key }, 'Failed to delete storage object on confirm error')
|
||||
}
|
||||
}
|
||||
|
||||
await this.prismaClient.asset.delete({ where: { id: asset.id } }).catch(() => {})
|
||||
if (asset.storageKeyId) {
|
||||
await this.prismaClient.storageKey
|
||||
.delete({ where: { id: asset.storageKeyId } })
|
||||
.catch(() => {})
|
||||
}
|
||||
const t = await this.prismaClient.task.findUnique({ where: { id: taskId } })
|
||||
if (t) {
|
||||
const updatedTask = await this.prismaClient.task.update({
|
||||
where: { id: taskId },
|
||||
data: { uploaded: { increment: 1 } },
|
||||
})
|
||||
if (updatedTask.uploaded === updatedTask.total) {
|
||||
await this.prismaClient.task.update({
|
||||
const discarded = await this.discardUploadingAsset(asset)
|
||||
if (discarded) {
|
||||
const t = await this.prismaClient.task.findUnique({ where: { id: taskId } })
|
||||
if (t) {
|
||||
const updatedTask = await this.prismaClient.task.update({
|
||||
where: { id: taskId },
|
||||
data: { status: TaskStatus.completed },
|
||||
data: { uploaded: { increment: 1 } },
|
||||
})
|
||||
if (updatedTask.uploaded === updatedTask.total) {
|
||||
await this.prismaClient.task.update({
|
||||
where: { id: taskId },
|
||||
data: { status: TaskStatus.completed },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (asset.status !== AssetStatus.uploading) {
|
||||
throw new Error('Asset is not in uploading status')
|
||||
}
|
||||
|
||||
const size = await s3Service.getObjectSize(process.env.S3_BUCKET || 'shumai', key)
|
||||
|
||||
const project = asset.project
|
||||
@@ -260,15 +247,22 @@ export class UploadService {
|
||||
resolvedMediaType = Bun.file(asset.name).type || 'application/octet-stream'
|
||||
}
|
||||
|
||||
// Update asset
|
||||
const updatedAsset = await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
// Claim status transition atomically
|
||||
const claim = await tx.asset.updateMany({
|
||||
where: {
|
||||
id: asset.id,
|
||||
taskId: taskId,
|
||||
status: AssetStatus.uploading,
|
||||
},
|
||||
data: {
|
||||
status: AssetStatus.uploaded,
|
||||
sizeByte: size,
|
||||
mediaType: resolvedMediaType,
|
||||
},
|
||||
})
|
||||
if (claim.count !== 1) {
|
||||
throw new Error('Asset is not in uploading status')
|
||||
}
|
||||
|
||||
// Update parent and ancestors
|
||||
if (asset.parentId) {
|
||||
@@ -277,7 +271,7 @@ export class UploadService {
|
||||
data: { fileCount: { increment: 1 } },
|
||||
})
|
||||
|
||||
await assetService.updateAncestorsSize(tx, asset.parentId, updatedAsset.sizeByte)
|
||||
await assetService.updateAncestorsSize(tx, asset.parentId, size)
|
||||
}
|
||||
|
||||
// Update task
|
||||
@@ -425,6 +419,7 @@ export class UploadService {
|
||||
total: t.total || 0,
|
||||
uploaded: t.uploaded,
|
||||
createdAt: t.createdAt.toISOString(),
|
||||
status: t.status,
|
||||
}))
|
||||
|
||||
return { data: infos, pageInfo }
|
||||
@@ -490,7 +485,6 @@ export class UploadService {
|
||||
taskId: string,
|
||||
req: AbortUploadRequest,
|
||||
): Promise<AbortUploadResponse> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const asset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: req.fileId },
|
||||
include: {
|
||||
@@ -520,33 +514,11 @@ export class UploadService {
|
||||
throw new Error('Provided key does not match asset storage key')
|
||||
}
|
||||
|
||||
const uploadId = asset.uploadId || req.uploadId
|
||||
if (asset.uploadId && req.uploadId && asset.uploadId !== req.uploadId) {
|
||||
throw new Error('Provided uploadId does not match asset uploadId')
|
||||
}
|
||||
|
||||
if (uploadId && key) {
|
||||
try {
|
||||
await s3Service.abortMultipartUpload(bucket, key, uploadId)
|
||||
} catch (err) {
|
||||
logger.warn({ err, key, uploadId }, 'Failed to abort multipart upload in S3')
|
||||
}
|
||||
} else if (key) {
|
||||
try {
|
||||
await s3Service.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
logger.warn({ err, key }, 'Failed to delete object from storage')
|
||||
}
|
||||
}
|
||||
|
||||
await this.prismaClient.asset.delete({ where: { id: asset.id } }).catch((err) => {
|
||||
logger.warn({ err, assetId: asset.id }, 'Failed to delete asset on upload abort')
|
||||
})
|
||||
if (asset.storageKeyId) {
|
||||
await this.prismaClient.storageKey
|
||||
.delete({ where: { id: asset.storageKeyId } })
|
||||
.catch(() => {})
|
||||
}
|
||||
await this.discardUploadingAsset(asset, req.uploadId)
|
||||
|
||||
if (taskId) {
|
||||
const remaining = await this.prismaClient.asset.count({
|
||||
@@ -564,6 +536,106 @@ export class UploadService {
|
||||
|
||||
return { success: true }
|
||||
}
|
||||
|
||||
private async discardUploadingAsset(
|
||||
asset: {
|
||||
id: string
|
||||
uploadId: string | null
|
||||
storageKeyId: string | null
|
||||
storageKey: { key: string } | null
|
||||
},
|
||||
requestedUploadId?: string,
|
||||
): Promise<boolean> {
|
||||
const deleteResult = await this.prismaClient.asset.deleteMany({
|
||||
where: {
|
||||
id: asset.id,
|
||||
status: AssetStatus.uploading,
|
||||
},
|
||||
})
|
||||
|
||||
if (deleteResult.count === 0) {
|
||||
return false
|
||||
}
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const key = asset.storageKey?.key
|
||||
const uploadId = asset.uploadId || requestedUploadId
|
||||
|
||||
if (uploadId && key) {
|
||||
try {
|
||||
await s3Service.abortMultipartUpload(bucket, key, uploadId)
|
||||
} catch (err) {
|
||||
logger.warn({ err, key, uploadId }, 'Failed to abort multipart upload in S3')
|
||||
}
|
||||
} else if (key) {
|
||||
try {
|
||||
await s3Service.deleteObject(bucket, key)
|
||||
} catch (err) {
|
||||
logger.warn({ err, key }, 'Failed to delete object from storage')
|
||||
}
|
||||
}
|
||||
|
||||
if (asset.storageKeyId) {
|
||||
await this.prismaClient.storageKey
|
||||
.delete({ where: { id: asset.storageKeyId } })
|
||||
.catch(() => {})
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async cancelUploadTask(
|
||||
teamId: string,
|
||||
userId: string,
|
||||
taskId: string,
|
||||
): Promise<CancelUploadTaskResponse> {
|
||||
const task = await this.prismaClient.task.findUnique({
|
||||
where: { id: taskId },
|
||||
include: {
|
||||
assets: {
|
||||
include: {
|
||||
storageKey: true,
|
||||
project: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if (!task) {
|
||||
throw new Error('Upload task not found')
|
||||
}
|
||||
|
||||
if (task.creatorId !== userId) {
|
||||
throw new Error('Unauthorized to cancel this upload task')
|
||||
}
|
||||
|
||||
if (task.status === TaskStatus.completed) {
|
||||
throw new Error('Cannot cancel a completed upload task')
|
||||
}
|
||||
|
||||
const invalidTeamAsset = task.assets.find(
|
||||
(a) => a.project?.teamId && a.project.teamId !== teamId,
|
||||
)
|
||||
if (invalidTeamAsset) {
|
||||
throw new Error('Task does not belong to specified team')
|
||||
}
|
||||
|
||||
const uploadingAssets = task.assets.filter((a) => a.status === AssetStatus.uploading)
|
||||
|
||||
for (const asset of uploadingAssets) {
|
||||
await this.discardUploadingAsset(asset)
|
||||
}
|
||||
|
||||
await this.prismaClient.task.updateMany({
|
||||
where: {
|
||||
id: taskId,
|
||||
status: { not: TaskStatus.completed },
|
||||
},
|
||||
data: { status: TaskStatus.failed },
|
||||
})
|
||||
|
||||
return { success: true }
|
||||
}
|
||||
}
|
||||
|
||||
export const uploadService = new UploadService()
|
||||
|
||||
@@ -547,4 +547,41 @@ describe('WorkflowTaskService', () => {
|
||||
expect(stopRes.task.fileSize).toBe(5242880)
|
||||
expect(stopRes.task.projectName).toBe('Enrich Project')
|
||||
})
|
||||
|
||||
it('reverts archiving and restoring asset status back to baseline on stopWorkflowTask', async () => {
|
||||
const team = await prisma.team.create({ data: { name: 'Stop Task Team' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Stop Project', teamId: team.id },
|
||||
})
|
||||
const sk = await prisma.storageKey.create({
|
||||
data: { key: 'keys/stop-test.mp4', isArchived: false },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'stop-test.mp4',
|
||||
type: 'file',
|
||||
status: AssetStatus.archiving,
|
||||
storageKeyId: sk.id,
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
teamId: team.id,
|
||||
projectId: project.id,
|
||||
type: WorkflowTaskType.archive_asset,
|
||||
status: WorkflowTaskStatus.processing,
|
||||
payload: { projectId: project.id, archive: { storageKeyId: sk.id } },
|
||||
},
|
||||
})
|
||||
|
||||
await workflowTaskService.stopWorkflowTask({ teamId: team.id, taskId: task.id })
|
||||
|
||||
const updatedAsset = await prisma.asset.findUniqueOrThrow({ where: { id: asset.id } })
|
||||
expect(updatedAsset.status).toBe(AssetStatus.processed)
|
||||
|
||||
const updatedTask = await prisma.workflowTask.findUniqueOrThrow({ where: { id: task.id } })
|
||||
expect(updatedTask.status).toBe(WorkflowTaskStatus.failed)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import type { WorkflowTaskItem, ListWorkflowTasksResponse } from '@shumai/dtos'
|
||||
import { paginateQuery } from '@shumai/core/src/pagination'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { revertArchiveStateForStorageKey } from '@shumai/core/src/asset/asset-archive'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
|
||||
export interface ListWorkflowTasksParams {
|
||||
@@ -300,15 +301,30 @@ export class WorkflowTaskService {
|
||||
}
|
||||
|
||||
if (task.assetId) {
|
||||
await tx.$executeRaw`
|
||||
UPDATE "assets"
|
||||
SET
|
||||
"status" = 'failed'::"AssetStatus",
|
||||
"media" = COALESCE("media", '{}'::jsonb) || jsonb_build_object('error', ${errorMsg}::text)
|
||||
WHERE "id" = ${task.assetId}
|
||||
AND "is_deleted" = false
|
||||
AND "status" != 'processed'::"AssetStatus"
|
||||
`
|
||||
if (task.type === 'archive_asset' || task.type === 'restore_asset') {
|
||||
const payload = task.payload as PrismaJson.WorkflowTaskPayload | null
|
||||
let storageKeyId = payload?.archive?.storageKeyId
|
||||
if (!storageKeyId) {
|
||||
const targetAsset = await tx.asset.findUnique({
|
||||
where: { id: task.assetId },
|
||||
select: { storageKeyId: true },
|
||||
})
|
||||
storageKeyId = targetAsset?.storageKeyId ?? undefined
|
||||
}
|
||||
if (storageKeyId) {
|
||||
await revertArchiveStateForStorageKey(storageKeyId, tx)
|
||||
}
|
||||
} else {
|
||||
await tx.$executeRaw`
|
||||
UPDATE "assets"
|
||||
SET
|
||||
"status" = 'failed'::"AssetStatus",
|
||||
"media" = COALESCE("media", '{}'::jsonb) || jsonb_build_object('error', ${errorMsg}::text)
|
||||
WHERE "id" = ${task.assetId}
|
||||
AND "is_deleted" = false
|
||||
AND "status" NOT IN ('processed'::"AssetStatus", 'archived'::"AssetStatus", 'archiving'::"AssetStatus", 'restoring'::"AssetStatus")
|
||||
`
|
||||
}
|
||||
}
|
||||
|
||||
return await tx.workflowTask.findUniqueOrThrow({
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/db",
|
||||
"version": "0.6.2",
|
||||
"version": "0.6.4",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
-- AlterEnum
|
||||
-- This migration adds more than one value to an enum.
|
||||
-- With PostgreSQL versions 11 and earlier, this is not possible
|
||||
-- in a single migration. This can be worked around by creating
|
||||
-- multiple migrations, each migration adding only one value to
|
||||
-- the enum.
|
||||
|
||||
|
||||
ALTER TYPE "AssetStatus" ADD VALUE 'archiving';
|
||||
ALTER TYPE "AssetStatus" ADD VALUE 'archived';
|
||||
ALTER TYPE "AssetStatus" ADD VALUE 'restoring';
|
||||
|
||||
-- AlterEnum
|
||||
-- This migration adds more than one value to an enum.
|
||||
-- With PostgreSQL versions 11 and earlier, this is not possible
|
||||
-- in a single migration. This can be worked around by creating
|
||||
-- multiple migrations, each migration adding only one value to
|
||||
-- the enum.
|
||||
|
||||
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'archive_asset';
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'restore_asset';
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "storage_keys" ADD COLUMN "is_archived" BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -40,6 +40,9 @@ enum AssetStatus {
|
||||
uploaded
|
||||
processing
|
||||
processed
|
||||
archiving
|
||||
archived
|
||||
restoring
|
||||
failed
|
||||
trashed
|
||||
pending_purge
|
||||
@@ -128,6 +131,8 @@ enum WorkflowTaskType {
|
||||
query_embedding_for_search
|
||||
agent_tool_call // @deprecated: Agent tools now execute directly in-process via Core services
|
||||
kanban_agent_run
|
||||
archive_asset
|
||||
restore_asset
|
||||
}
|
||||
|
||||
enum WorkflowTaskStatus {
|
||||
@@ -606,12 +611,13 @@ model Asset {
|
||||
}
|
||||
|
||||
model StorageKey {
|
||||
id String @id @default(ulid())
|
||||
key String @unique
|
||||
status StorageKeyStatus @default(active)
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
assets Asset[]
|
||||
id String @id @default(ulid())
|
||||
key String @unique
|
||||
status StorageKeyStatus @default(active)
|
||||
isArchived Boolean @default(false) @map("is_archived")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
assets Asset[]
|
||||
|
||||
@@map("storage_keys")
|
||||
}
|
||||
|
||||
+6
-1
@@ -36,6 +36,9 @@ export const AssetStatus = {
|
||||
uploaded: 'uploaded',
|
||||
processing: 'processing',
|
||||
processed: 'processed',
|
||||
archiving: 'archiving',
|
||||
archived: 'archived',
|
||||
restoring: 'restoring',
|
||||
failed: 'failed',
|
||||
trashed: 'trashed',
|
||||
pending_purge: 'pending_purge'
|
||||
@@ -156,7 +159,9 @@ export const WorkflowTaskType = {
|
||||
ai_embedding: 'ai_embedding',
|
||||
query_embedding_for_search: 'query_embedding_for_search',
|
||||
agent_tool_call: 'agent_tool_call',
|
||||
kanban_agent_run: 'kanban_agent_run'
|
||||
kanban_agent_run: 'kanban_agent_run',
|
||||
archive_asset: 'archive_asset',
|
||||
restore_asset: 'restore_asset'
|
||||
} as const
|
||||
|
||||
export type WorkflowTaskType = (typeof WorkflowTaskType)[keyof typeof WorkflowTaskType]
|
||||
|
||||
+4
-4
File diff suppressed because one or more lines are too long
@@ -5201,6 +5201,7 @@ export const StorageKeyScalarFieldEnum = {
|
||||
id: 'id',
|
||||
key: 'key',
|
||||
status: 'status',
|
||||
isArchived: 'isArchived',
|
||||
createdAt: 'createdAt',
|
||||
updatedAt: 'updatedAt'
|
||||
} as const
|
||||
|
||||
@@ -375,6 +375,7 @@ export const StorageKeyScalarFieldEnum = {
|
||||
id: 'id',
|
||||
key: 'key',
|
||||
status: 'status',
|
||||
isArchived: 'isArchived',
|
||||
createdAt: 'createdAt',
|
||||
updatedAt: 'updatedAt'
|
||||
} as const
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user