Compare commits

...
58 Commits
Author SHA1 Message Date
yiling b6afcc135e Release v0.2.0 2026-07-21 21:30:00 +08:00
Yiling-J fb28b7d020 Merge pull request #208 from shumaiOne/feat/agent-media-tools-refactor
feat(agent): refactor media tools to static injection with mandatory assetId and authz
2026-07-21 16:33:12 +08:00
yiling 28c3df3429 fix(agent): require mandatory userId for media tools and optimize authz 2026-07-21 16:22:14 +08:00
yiling 93e957fa4f feat(agent): refactor media tools to static injection with mandatory assetId and authz 2026-07-21 16:09:39 +08:00
Yiling-J 1ac0d1452e Merge pull request #207 from shumaiOne/fix/version-stack-agent-chat
fix(agent): pass latest version asset from version stack to prompt builder
2026-07-21 15:16:55 +08:00
yiling 5855134236 fix(agent): pass latest version asset from version stack to prompt builder 2026-07-21 15:04:38 +08:00
Yiling-J 581a12bdc9 Merge pull request #206 from shumaiOne/fix/latest-children-trashed-files
fix(asset): exclude soft-deleted children from latestChildren preview
2026-07-21 11:33:37 +08:00
yiling 851e5d975c fix(asset): exclude soft-deleted children from latestChildren preview 2026-07-21 11:19:20 +08:00
Yiling-J 6e69fc451b Merge pull request #205 from shumaiOne/fix/include-skill-id-in-prompt
fix(agent): include skill id in formatSkillsForPrompt
2026-07-21 10:46:39 +08:00
yiling b91033495c fix(agent): include skill id in formatSkillsForPrompt 2026-07-21 10:42:01 +08:00
Yiling-J a2895cd5d1 Merge pull request #204 from shumaiOne/feat/chat-context-asset-change
feat(chat): track context asset changes and update agent context
2026-07-21 10:13:09 +08:00
yiling e3352232fe feat(chat): track context asset changes and update agent context 2026-07-21 10:01:22 +08:00
Yiling-J 6d34b13266 Merge pull request #203 from shumaiOne/feat/markdown-proxy
feat(transcode): add markdown file to pdf proxy support
2026-07-20 18:37:54 +08:00
yiling 208eb6898a feat(transcode): add markdown file to pdf proxy support 2026-07-20 18:35:16 +08:00
Yiling-J b1d9c1b31b Merge pull request #202 from shumaiOne/add-fonts-noto-cjk-to-dockerfile
feat(docker): install fonts-noto-cjk in runner stage
2026-07-20 16:36:18 +08:00
yiling c016d1dffe feat(docker): install fonts-noto-cjk in runner stage 2026-07-20 16:35:19 +08:00
Yiling-J 7cd8a5bef7 Merge pull request #201 from shumaiOne/fix/transcode-pdf-font
fix(transcode): mark pdfkit as external and extract for docker runtime
2026-07-20 16:16:05 +08:00
yiling 3274d19574 fix(transcode): mark pdfkit as external and extract for docker runtime 2026-07-20 16:12:59 +08:00
Yiling-J 047174c13d Add Discord link to README 2026-07-20 15:05:43 +08:00
Yiling-J d9d7b52758 Merge pull request #200 from shumaiOne/feat/use-proxy-type
feat(asset): replace mediaType/mimeType logic with proxyType
2026-07-20 14:59:39 +08:00
yiling 52edcd1a79 fix(transcode): fix unused import and set proxyType in transcode workflow tests 2026-07-20 14:50:43 +08:00
yiling 3afe0fc3c3 feat(api): remove mediaType and mimeType from DTO responses and DB MediaInfo interface 2026-07-20 14:37:17 +08:00
yiling 5d4020dc2d fix(mime): restrict pdf proxyType strictly to pdf, csv, and txt files 2026-07-20 11:50:19 +08:00
yiling 4ad0df5876 fix(agent): block non-pdf assets in read_pdf_pages and remove mediaType fallback in AgentChatPromptBuilder 2026-07-20 11:01:22 +08:00
yiling fea153e94f fix(asset): resolve DTO schema and test typecheck errors for proxyType 2026-07-20 10:46:20 +08:00
yiling f158d04686 feat(asset): replace mediaType/mimeType logic with proxyType 2026-07-20 10:22:36 +08:00
Yiling-J c45a1d9ee9 Merge pull request #199 from shumaiOne/feat/txt-csv-pdf-proxy
feat(transcode): add PDF proxy support for TXT and CSV files
2026-07-20 09:41:28 +08:00
yiling 29995a64fd fix(transcode): add fontName to Linux TTC fonts, Windows CJK paths, and Droid fallback 2026-07-20 09:35:49 +08:00
yiling 1970535aa6 fix(transcode): use csv-parse for CSV parsing and pin pdfkit exact version 2026-07-19 22:46:13 +08:00
yiling 7574dd56e4 feat(db): add backfill migration for media.proxyType 2026-07-19 21:21:31 +08:00
yiling 081dbf6d39 feat(transcode): add PDF proxy support for TXT and CSV files 2026-07-19 21:19:43 +08:00
Yiling-J 817889e501 Merge pull request #197 from shumaiOne/feat/split-transcode-workflows
feat(transcode): split monolithic transcodeMedia into focused workflows
2026-07-19 15:01:38 +08:00
yiling aa572bf2d6 docs(transcode): add deprecation notice to legacy transcodeMedia workflow 2026-07-19 14:50:31 +08:00
yiling d9ed30fedb ci: install poppler-utils and ffmpeg system dependencies for workflow E2E tests 2026-07-19 14:43:04 +08:00
yiling 7d7a1b61c0 test(e2e): add E2E test for renderPdfPagesWorkflow 2026-07-19 13:27:07 +08:00
yiling cbe622dfeb feat(transcode): split monolithic transcodeMedia into focused workflows 2026-07-19 12:32:08 +08:00
Yiling-J b72a9435e7 Merge pull request #196 from shumaiOne/feat/agent-read-pdf-pages
feat(agent): add read_pdf_pages tool and PDF page context support
2026-07-19 12:14:20 +08:00
yiling 759e37915b refactor(transcode): add overlayAnnotationsOnBuffer and add pixel verification tests 2026-07-19 12:03:23 +08:00
yiling e2523e1a81 perf(transcode): optimize PDF page rendering pipeline using in-memory sharp buffers 2026-07-19 10:18:50 +08:00
yiling 6064a1449a fix(agent): enforce page range validation and refine pdftoppm error check 2026-07-19 10:17:21 +08:00
yiling f68ba70656 feat(agent): add read_pdf_pages tool and PDF page context support 2026-07-19 09:45:14 +08:00
Yiling-J 68a0fb7319 Merge pull request #195 from shumaiOne/feat/pdf-worker-route
feat(web): register pdf.worker route in production
2026-07-18 19:44:43 +08:00
yiling 661ec06cef fix(webui): ensure pdf.worker path is root-relative 2026-07-18 19:14:56 +08:00
yiling 6000b8e366 feat(web): register pdf.worker route manually in prod mode 2026-07-18 19:10:07 +08:00
Yiling-J db3551de1d Merge pull request #193 from shumaiOne/feat/pdf-support
feat(transcode): add PDF preview, sprite generation, and annotation support
2026-07-18 15:42:15 +08:00
yiling dc20c7f548 test(e2e): add PDF workflow E2E test case using test.pdf fixture 2026-07-18 15:33:05 +08:00
yiling f76e7a8e8d fix(webui): cancel renderTask immediately if active flag is false before or during render 2026-07-18 15:17:38 +08:00
yiling cfdc085ae7 chore(deps): move pdfjs-dist to webui package.json and clean root dependencies 2026-07-18 15:15:45 +08:00
yiling 779706a3e7 test(core): add unit tests for generatePdfSprite and getPdfInfo in TranscodeService 2026-07-18 15:07:28 +08:00
yiling 06dc929a2a fix(webui): destroy loadingTask and cancel renderTask on unmount 2026-07-18 14:05:10 +08:00
yiling 0ad3aca083 fix(webui): add type declaration for *.mjs worker imports 2026-07-18 14:03:31 +08:00
yiling b59f439683 feat(webui): self-host pdf.worker.min.mjs in public directory 2026-07-18 13:08:55 +08:00
yiling d270f3f62c fix(webui): add keyboard navigation shortcuts for PDF viewer 2026-07-18 11:25:59 +08:00
yiling ea201ad19f feat(webui): support custom timestamp formatting for PDF viewer comments (P9) 2026-07-18 11:20:13 +08:00
yiling 2b72f3038a fix(webui): enable hasTimestamp for PDF viewer comments configuration 2026-07-18 10:55:29 +08:00
yiling c1ddbe4be7 fix(webui): sync pdf page with seekTo and onTimeUpdate 2026-07-18 10:49:53 +08:00
yiling 3ada194a8e fix(asset): presign poster thumbnail for PDF and poster-enabled assets 2026-07-18 10:41:28 +08:00
yiling bc1491bed5 feat(transcode): add PDF preview, sprite generation, and annotation support 2026-07-18 10:32:38 +08:00
144 changed files with 64465 additions and 1070 deletions
+2 -2
View File
@@ -32,7 +32,7 @@ jobs:
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ripgrep bubblewrap socat
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils
- name: Lint
run: bun run lint
@@ -157,7 +157,7 @@ jobs:
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils
- name: Run Workflow E2E Tests
run: bun run test:e2e:workflow
+1 -1
View File
@@ -48,7 +48,7 @@ jobs:
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y ripgrep bubblewrap socat libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find
- name: Install dependencies
run: bun install
+2 -1
View File
@@ -141,7 +141,7 @@ We follow Bun's monorepo conventions for dependency management:
3. **Local DevDependencies**: Tools specific to a single package (e.g., `@vitejs/plugin-react` for `webui`) should be declared in that package's local `package.json`.
4. **Workspace Imports**: Use `workspace:*` for internal package dependencies.
5. **Clean Root**: The root `package.json` must not contain runtime `dependencies`. It is reserved for shared `devDependencies` and workspace-wide `scripts`.
6. **Installation**: Always run `bun install` from the root. Use the `--filter` flag if you only want to update a specific package (e.g., `bun install --filter @shumai/webui`).
6. **Adding Packages**: Never edit `package.json` manually to add new dependencies. Always use `bun add` (e.g., `bun add <package> --filter <workspace>`) from the root without specifying an explicit version, letting Bun resolve the correct version.
## TypeScript Configuration
@@ -314,6 +314,7 @@ We use `prisma-json-types-generator` to enforce strict type-safety for Prisma `J
- Development: Use `bun --bun run prisma migrate dev` to create and apply migrations during development.
- Production: Use `bun --bun run prisma migrate deploy` to apply pending migrations in production environments.
- **No Manual Migration Creation**: Never create migration SQL files or directories manually by hand. Always use Prisma CLI commands (e.g. `bun --bun run prisma migrate dev --create-only` to generate a migration template, or `bun --bun run prisma migrate dev`) so Prisma correctly tracks migration metadata and checksums.
- **No Automatic Dev DB Reset**: Do not run `prisma migrate reset --force` or commands that force-reset the database automatically. If migrations become out of sync or a reset is required, stop executing, report the situation to the user, and present suggested manual cleanup/reset steps for the user to execute.
### Commands
+2
View File
@@ -49,6 +49,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
bubblewrap \
gosu \
poppler-utils \
fonts-noto-cjk \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
+7 -5
View File
@@ -9,7 +9,7 @@
</p>
<p align="center">
<a target="_blank" href="https://shumai.one/">Website</a> | <a target="_blank" href="https://docs.shumai.one/introduction">Docs</a> | <a target="_blank" href="https://staging.shumai.one">Demo</a> | <a target="_blank" href="README_CN.md">中文</a>
<a target="_blank" href="https://shumai.one/">Website</a> | <a target="_blank" href="https://docs.shumai.one/introduction">Docs</a> | <a target="_blank" href="https://staging.shumai.one">Demo</a> | <a target="_blank" href="https://discord.gg/sUzhujK6DD">Discord</a> | <a target="_blank" href="README_CN.md">中文</a>
</p>
![Shumai App Screenshot](docs/screenshot.webp)
@@ -125,23 +125,24 @@ mkdir shumai && cd shumai
Shumai requires the following system packages on Linux:
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
- **`poppler`** (`poppler-utils`) – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
- **`bubblewrap`**, **`socat`**, and **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`) for process isolation, networking, and workspace search.
Install all required packages with one command:
> Ubuntu/Debian
```bash
sudo apt install -y ffmpeg bubblewrap socat ripgrep
sudo apt install -y ffmpeg poppler-utils bubblewrap socat ripgrep
```
> Fedora
```bash
sudo dnf install -y ffmpeg bubblewrap socat ripgrep
sudo dnf install -y ffmpeg poppler-utils bubblewrap socat ripgrep
```
> Arch Linux
```bash
sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
sudo pacman -S --noconfirm ffmpeg poppler bubblewrap socat ripgrep
```
> [!NOTE]
@@ -162,12 +163,13 @@ sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
Shumai requires:
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
- **`poppler`** – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
- **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`).
Install the required packages with Homebrew:
```bash
brew install ffmpeg ripgrep
brew install ffmpeg poppler ripgrep
```
---
+6 -4
View File
@@ -139,23 +139,24 @@ mkdir shumai && cd shumai
在 Linux 上运行 Shumai 需要安装以下系统依赖:
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
- **`poppler`** (`poppler-utils`) —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
- **`bubblewrap`**、**`socat`** 和 **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需,用于进程隔离、网络通信和工作区搜索。
可以使用以下命令一次性安装所有依赖:
> Ubuntu/Debian
```bash
sudo apt install -y ffmpeg bubblewrap socat ripgrep
sudo apt install -y ffmpeg poppler-utils bubblewrap socat ripgrep
```
> Fedora
```bash
sudo dnf install -y ffmpeg bubblewrap socat ripgrep
sudo dnf install -y ffmpeg poppler-utils bubblewrap socat ripgrep
```
> Arch Linux
```bash
sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
sudo pacman -S --noconfirm ffmpeg poppler bubblewrap socat ripgrep
```
> [!NOTE]
@@ -176,12 +177,13 @@ sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
在 macOS 上运行 Shumai 需要:
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
- **`poppler`** —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
- **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需。
使用 Homebrew 安装所需依赖:
```bash
brew install ffmpeg ripgrep
brew install ffmpeg poppler ripgrep
```
---
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/agent-app",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/cli",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"bin": {
+2
View File
@@ -33,6 +33,8 @@ export function getMediaType(filename: string): string {
'.wav': 'audio/wav',
'.ogg': 'audio/ogg',
'.txt': 'text/plain',
'.md': 'text/markdown',
'.markdown': 'text/markdown',
'.json': 'application/json',
'.pdf': 'application/pdf',
'.zip': 'application/zip',
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/transcode-app",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/web-app",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"dependencies": {
+23 -1
View File
@@ -1,5 +1,6 @@
import { loadEnvConfig } from '@shumai/core/src/env-loader'
import { join } from 'node:path'
import { readdirSync } from 'node:fs'
import { dirname, join } from 'node:path'
loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
import index from '@shumai/webui/index.html'
@@ -84,6 +85,27 @@ async function run() {
routes[routePath] = () => new Response(Bun.file(filePath), { headers: file.headers })
}
const sampleFile = bundle.files[0]?.path || bundle.index || ''
const bundleDir = join(import.meta.dir, dirname(sampleFile))
try {
const dirFiles = readdirSync(bundleDir)
const pdfWorkerFile = dirFiles.find((f) => f.startsWith('pdf.worker'))
if (pdfWorkerFile) {
const routePath = '/' + pdfWorkerFile
const filePath = join(bundleDir, pdfWorkerFile)
const bunFile = Bun.file(filePath)
routes[routePath] = () =>
new Response(bunFile, {
headers: {
'content-type': 'text/javascript;charset=utf-8',
etag: `"${bunFile.size}-${bunFile.lastModified}"`,
},
})
}
} catch (err) {
console.warn('Failed to register pdf.worker route:', err)
}
const mainHtmlPath = join(import.meta.dir, bundle.index)
const mainHtmlFile = Bun.file(mainHtmlPath)
const mainHtmlHeaders = bundle.files.find((f) => f.path === bundle.index)?.headers || {
+1
View File
@@ -34,6 +34,7 @@ await Bun.build({
'@temporalio/client',
'@temporalio/worker',
'@temporalio/workflow',
'pdfkit',
],
// compile: {
// outfile: 'Shumai',
+87 -18
View File
@@ -4,10 +4,6 @@
"workspaces": {
"": {
"name": "shumai",
"dependencies": {
"@tanstack/react-virtual": "3.14.2",
"jittered-fractional-indexing": "1.0.1",
},
"devDependencies": {
"@eslint/js": "10.0.1",
"@inlang/paraglide-js": "2.20.0",
@@ -37,7 +33,7 @@
},
"apps/agent": {
"name": "@shumai/agent-app",
"version": "0.1.9",
"version": "0.1.10",
"bin": {
"shumai-agent": "./src/index.ts",
},
@@ -49,7 +45,7 @@
},
"apps/cli": {
"name": "@shumai/cli",
"version": "0.1.9",
"version": "0.1.10",
"bin": {
"shumai-cli": "./src/index.ts",
},
@@ -64,7 +60,7 @@
},
"apps/transcode": {
"name": "@shumai/transcode-app",
"version": "0.1.9",
"version": "0.1.10",
"bin": {
"shumai-transcode": "./src/index.ts",
},
@@ -76,7 +72,7 @@
},
"apps/web": {
"name": "@shumai/web-app",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@shumai/agent": "workspace:*",
"@shumai/api": "workspace:*",
@@ -96,7 +92,7 @@
},
"packages/agent": {
"name": "@shumai/agent",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@anthropic-ai/sandbox-runtime": "0.0.52",
"@earendil-works/pi-agent-core": "0.78.0",
@@ -111,6 +107,7 @@
"@temporalio/worker": "1.17.2",
"@temporalio/workflow": "1.17.2",
"adm-zip": "0.5.17",
"jittered-fractional-indexing": "1.0.1",
"ulid": "3.0.2",
"zod": "4.4.3",
},
@@ -120,7 +117,7 @@
},
"packages/api": {
"name": "@shumai/api",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@hono/zod-validator": "0.7.6",
"@shumai/core": "workspace:*",
@@ -134,7 +131,7 @@
},
"packages/core": {
"name": "@shumai/core",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@shumai/db": "workspace:*",
@@ -142,12 +139,14 @@
"@sinclair/typebox": "0.34.49",
"adm-zip": "0.5.17",
"better-auth": "1.6.11",
"csv-parse": "7.0.1",
"date-fns": "4.4.0",
"dotenv": "16.6.1",
"dotenv-expand": "13.0.0",
"extract-zip": "2.0.1",
"hono": "4.12.23",
"jittered-fractional-indexing": "1.0.1",
"pdfkit": "0.19.1",
"pino": "10.3.1",
"sharp": "0.34.5",
"tweetnacl": "1.0.3",
@@ -158,11 +157,12 @@
"devDependencies": {
"@types/adm-zip": "0.5.8",
"@types/extract-zip": "2.0.3",
"@types/pdfkit": "0.17.6",
},
},
"packages/db": {
"name": "@shumai/db",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@prisma/adapter-pg": "7.8.0",
@@ -178,7 +178,7 @@
},
"packages/dtos": {
"name": "@shumai/dtos",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"zod": "4.4.3",
@@ -186,7 +186,7 @@
},
"packages/e2e": {
"name": "@shumai/e2e",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@google/genai": "1.52.0",
@@ -200,11 +200,11 @@
},
"packages/tableprinter": {
"name": "@shumai/tableprinter",
"version": "0.1.9",
"version": "0.1.10",
},
"packages/transcode": {
"name": "@shumai/transcode",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -218,7 +218,7 @@
},
"packages/webui": {
"name": "@shumai/webui",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@base-ui/react": "1.5.0",
"@dnd-kit/abstract": "0.4.0",
@@ -280,6 +280,7 @@
"lexical": "0.42.0",
"lucide-react": "1.17.0",
"next-themes": "0.4.6",
"pdfjs-dist": "6.1.200",
"radix-ui": "1.4.3",
"react": "19.2.0",
"react-day-picker": "10.0.1",
@@ -315,7 +316,7 @@
},
"packages/workflow-core": {
"name": "@shumai/workflow-core",
"version": "0.1.9",
"version": "0.1.10",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -732,6 +733,30 @@
"@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="],
"@napi-rs/canvas": ["@napi-rs/canvas@1.0.2", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.2", "@napi-rs/canvas-darwin-arm64": "1.0.2", "@napi-rs/canvas-darwin-x64": "1.0.2", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", "@napi-rs/canvas-linux-arm64-musl": "1.0.2", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-musl": "1.0.2", "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", "@napi-rs/canvas-win32-x64-msvc": "1.0.2" } }, "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ=="],
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.2", "", { "os": "android", "cpu": "arm64" }, "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g=="],
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ=="],
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A=="],
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ=="],
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA=="],
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA=="],
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw=="],
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g=="],
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q=="],
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ=="],
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A=="],
"@noble/ciphers": ["@noble/ciphers@2.2.0", "", {}, "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA=="],
"@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="],
@@ -1092,6 +1117,8 @@
"@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="],
"@swc/helpers": ["@swc/helpers@0.5.23", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw=="],
"@swc/types": ["@swc/types@0.1.26", "", { "dependencies": { "@swc/counter": "^0.1.3" } }, "sha512-lyMwd7WGgG79RS7EERZV3T8wMdmPq3xwyg+1nmAM64kIhx5yl+juO2PYIHb7vTiPgPCj8LYjsNV2T5wiQHUEaw=="],
"@tailwindcss/node": ["@tailwindcss/node@4.3.0", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.21.0", "jiti": "^2.6.1", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.0" } }, "sha512-aFb4gUhFOgdh9AXo4IzBEOzBkkAxm9VigwDJnMIYv3lcfXCJVesNfbEaBl4BNgVRyid92AmdviqwBUBRKSeY3g=="],
@@ -1238,6 +1265,8 @@
"@types/node": ["@types/node@25.9.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg=="],
"@types/pdfkit": ["@types/pdfkit@0.17.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-tIwzxk2uWKp0Cq9JIluQXJid77lYhF52EsIOwhsMF4iWLA6YneoBR1xVKYYdAysHuepUB0OX4tdwMiUDdGKmig=="],
"@types/pg": ["@types/pg@8.20.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow=="],
"@types/react": ["@types/react@19.2.16", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-esJiCAnl0kfpNdE69f3So4WJUXy95dLZydX0KwK46riIHDzHM7O9Vtf9xCHW0PXIqvgqNrswl522kA/5yx+F4w=="],
@@ -1448,6 +1477,10 @@
"brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="],
"brotli": ["brotli@1.3.3", "", { "dependencies": { "base64-js": "^1.1.2" } }, "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg=="],
"browserify-zlib": ["browserify-zlib@0.2.0", "", { "dependencies": { "pako": "~1.0.5" } }, "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA=="],
"browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="],
"buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="],
@@ -1506,6 +1539,8 @@
"cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="],
"clone": ["clone@2.1.2", "", {}, "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w=="],
"clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
"cmdk": ["cmdk@1.1.1", "", { "dependencies": { "@radix-ui/react-compose-refs": "^1.1.1", "@radix-ui/react-dialog": "^1.1.6", "@radix-ui/react-id": "^1.1.0", "@radix-ui/react-primitive": "^2.0.2" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, "sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg=="],
@@ -1546,6 +1581,8 @@
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
"csv-parse": ["csv-parse@7.0.1", "", {}, "sha512-+2z7Ar0APQ7Uu6fX4cn+pitRmxjZ1WPBcGmZFKmA74FCyi7Et/XZx8cjNQ5CjbZ4HCOxXCOpRBYvYH08Qa003A=="],
"d3-array": ["d3-array@3.2.4", "", { "dependencies": { "internmap": "1 - 2" } }, "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg=="],
"d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="],
@@ -1610,6 +1647,8 @@
"devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="],
"dfa": ["dfa@1.2.0", "", {}, "sha512-ED3jP8saaweFTjeGX8HQPjeC1YYyZs98jGNZx6IiBvxW7JG5v492kamAQB3m2wop07CvU/RQmzcKr6bgcC5D/Q=="],
"diff": ["diff@8.0.4", "", {}, "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw=="],
"docker-compose": ["docker-compose@1.4.2", "", { "dependencies": { "yaml": "^2.2.2" } }, "sha512-rPHigTKGaEHpkUmfd69QgaOp+Os5vGJwG/Ry8lcr8W/382AmI+z/D7qoa9BybKIkqNppaIbs8RYeHSevdQjWww=="],
@@ -1762,6 +1801,8 @@
"flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="],
"fontkit": ["fontkit@2.0.4", "", { "dependencies": { "@swc/helpers": "^0.5.12", "brotli": "^1.3.2", "clone": "^2.1.2", "dfa": "^1.2.0", "fast-deep-equal": "^3.1.3", "restructure": "^3.0.0", "tiny-inflate": "^1.0.3", "unicode-properties": "^1.4.0", "unicode-trie": "^2.0.0" } }, "sha512-syetQadaUEDNdxdugga9CpEYVaQIxOwk7GlwZWWZ19//qW4zE5bknOKeMBDYAASwnpaSHKJITRLMF9m1fp3s6g=="],
"for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="],
"foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="],
@@ -1988,6 +2029,8 @@
"joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="],
"js-md5": ["js-md5@0.8.3", "", {}, "sha512-qR0HB5uP6wCuRMrWPTrkMaev7MJZwJuuw4fnwAzRgP4J4/F8RwtodOKpGp4XpqsLBFzzgqIO42efFAyz2Et6KQ=="],
"js-sha256": ["js-sha256@0.11.1", "", {}, "sha512-o6WSo/LUvY2uC4j7mO50a2ms7E/EAdbP0swigLV+nzHKTTaYnaLIWJ02VdXrsJX0vGedDESQnLsOekr94ryfjg=="],
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
@@ -2050,6 +2093,8 @@
"lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="],
"linebreak": ["linebreak@1.1.0", "", { "dependencies": { "base64-js": "0.0.8", "unicode-trie": "^2.0.0" } }, "sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ=="],
"loader-runner": ["loader-runner@4.3.2", "", {}, "sha512-DFEqQ3ihfS9blba08cLfYf1NRAIEm+dDjic073DRDc3/JspI/8wYmtDsHwd3+4hwvdxSK7PGaElfTmm0awWJ4w=="],
"locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="],
@@ -2226,6 +2271,8 @@
"package-json-from-dist": ["package-json-from-dist@1.0.1", "", {}, "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="],
"pako": ["pako@0.2.9", "", {}, "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA=="],
"parse-entities": ["parse-entities@4.0.2", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="],
"partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="],
@@ -2242,6 +2289,10 @@
"pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
"pdfjs-dist": ["pdfjs-dist@6.1.200", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.0" } }, "sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw=="],
"pdfkit": ["pdfkit@0.19.1", "", { "dependencies": { "@noble/ciphers": "^1.0.0", "@noble/hashes": "^1.6.0", "fontkit": "^2.0.4", "js-md5": "^0.8.3", "linebreak": "^1.1.0", "png-js": "^1.1.0" } }, "sha512-6Gzk+wDwTs4VSxsR5rCMTnIl5nlmkye1oWB0l2hDB1EX6ZNSIBroKQEv+2+fPPn+stVjyqzmsqRJVDfB9fo5DA=="],
"pend": ["pend@1.2.0", "", {}, "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="],
"perfect-debounce": ["perfect-debounce@2.1.0", "", {}, "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g=="],
@@ -2282,6 +2333,8 @@
"playwright-core": ["playwright-core@1.61.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg=="],
"png-js": ["png-js@1.1.0", "", { "dependencies": { "browserify-zlib": "^0.2.0" } }, "sha512-PM/uYGzGdNSzqeOgly68+6wKQDL1SY0a/N+OEa/+br6LnHWOAJB0Npiamnodfq3jd2LS/i2fMeOKSAILjA+m5Q=="],
"possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="],
"postcss": ["postcss@8.5.15", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A=="],
@@ -2404,6 +2457,8 @@
"resolve": ["resolve@2.0.0-next.7", "", { "dependencies": { "es-errors": "^1.3.0", "is-core-module": "^2.16.2", "node-exports-info": "^1.6.0", "object-keys": "^1.1.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-tqt+NBWwyaMgw3zDsnygx4CByWjQEJHOPMdslYhppaQSJUtL/D4JO9CcBBlhPoI8lz9oJIDXkwXfhF4aWqP8xQ=="],
"restructure": ["restructure@3.0.2", "", {}, "sha512-gSfoiOEA0VPE6Tukkrr7I0RBdE0s7H1eFCDBk05l1KIQT1UIKNc5JZy6jdyW6eYH3aR3g5b3PuL77rq0hvwtAw=="],
"retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="],
"rollup": ["rollup@4.61.0", "", { "dependencies": { "@types/estree": "1.0.9" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.61.0", "@rollup/rollup-android-arm64": "4.61.0", "@rollup/rollup-darwin-arm64": "4.61.0", "@rollup/rollup-darwin-x64": "4.61.0", "@rollup/rollup-freebsd-arm64": "4.61.0", "@rollup/rollup-freebsd-x64": "4.61.0", "@rollup/rollup-linux-arm-gnueabihf": "4.61.0", "@rollup/rollup-linux-arm-musleabihf": "4.61.0", "@rollup/rollup-linux-arm64-gnu": "4.61.0", "@rollup/rollup-linux-arm64-musl": "4.61.0", "@rollup/rollup-linux-loong64-gnu": "4.61.0", "@rollup/rollup-linux-loong64-musl": "4.61.0", "@rollup/rollup-linux-ppc64-gnu": "4.61.0", "@rollup/rollup-linux-ppc64-musl": "4.61.0", "@rollup/rollup-linux-riscv64-gnu": "4.61.0", "@rollup/rollup-linux-riscv64-musl": "4.61.0", "@rollup/rollup-linux-s390x-gnu": "4.61.0", "@rollup/rollup-linux-x64-gnu": "4.61.0", "@rollup/rollup-linux-x64-musl": "4.61.0", "@rollup/rollup-openbsd-x64": "4.61.0", "@rollup/rollup-openharmony-arm64": "4.61.0", "@rollup/rollup-win32-arm64-msvc": "4.61.0", "@rollup/rollup-win32-ia32-msvc": "4.61.0", "@rollup/rollup-win32-x64-gnu": "4.61.0", "@rollup/rollup-win32-x64-msvc": "4.61.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-T9mWdbWfQtp0B5lv/HX+wrhYsmXRlcWnXXmJbXqKJhlRaoS6KMhq0gpyzW4UJfclcxrEdLnTgjT2NjruLONu0g=="],
@@ -2566,6 +2621,8 @@
"thread-stream": ["thread-stream@4.2.0", "", { "dependencies": { "real-require": "^1.0.0" } }, "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ=="],
"tiny-inflate": ["tiny-inflate@1.0.3", "", {}, "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw=="],
"tiny-invariant": ["tiny-invariant@1.3.3", "", {}, "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="],
"tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="],
@@ -2622,6 +2679,10 @@
"undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],
"unicode-properties": ["unicode-properties@1.4.1", "", { "dependencies": { "base64-js": "^1.3.0", "unicode-trie": "^2.0.0" } }, "sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg=="],
"unicode-trie": ["unicode-trie@2.0.0", "", { "dependencies": { "pako": "^0.2.5", "tiny-inflate": "^1.0.0" } }, "sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ=="],
"unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="],
"unionfs": ["unionfs@4.6.0", "", { "dependencies": { "fs-monkey": "^1.0.0" } }, "sha512-fJAy3gTHjFi5S3TP5EGdjs/OUMFFvI/ady3T8qVuZfkv8Qi8prV/Q8BuFEgODJslhZTT2z2qdD2lGdee9qjEnA=="],
@@ -3026,6 +3087,8 @@
"bl/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="],
"browserify-zlib/pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
"bun-types/@types/node": ["@types/node@22.19.19", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew=="],
"c12/dotenv": ["dotenv@17.4.2", "", {}, "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw=="],
@@ -3054,12 +3117,18 @@
"lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
"linebreak/base64-js": ["base64-js@0.0.8", "", {}, "sha512-3XSA2cR/h/73EzlXXdU6YNycmYI7+kicTxks4eJg2g39biHR84slg2+des+p7iHYhbRg/udIS4TD53WabcOUkw=="],
"p-retry/retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="],
"parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="],
"path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
"pdfkit/@noble/ciphers": ["@noble/ciphers@1.3.0", "", {}, "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw=="],
"pdfkit/@noble/hashes": ["@noble/hashes@1.8.0", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="],
"pg-types/postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="],
"pretty-format/react-is": ["react-is@17.0.2", "", {}, "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="],
+14 -2
View File
@@ -3,12 +3,15 @@ title: Agent Tools
description: Configure the tools and capabilities permitted for each agent.
---
Shumai agents are equipped with built-in tools that allow them to interact with the file system, view files, extract video frames, and execute commands.
Shumai agents are equipped with built-in tools that allow them to interact with the file system, view files, extract video frames, and execute commands.
By default, **all tools are enabled** for new agents. Team Owners can deny specific tools to restrict agent capabilities.
<Warning>
**Caution:** Disabling tools can cause agents or skills to fail. For example, disabling the **Sandboxed Terminal** tool will break almost all custom skills that rely on executing commands or script environments. Disable tools only when you have a specific security policy or isolation requirement.
**Caution:** Disabling tools can cause agents or skills to fail. For example, disabling the
**Sandboxed Terminal** tool will break almost all custom skills that rely on executing commands or
script environments. Disable tools only when you have a specific security policy or isolation
requirement.
</Warning>
---
@@ -18,34 +21,42 @@ By default, **all tools are enabled** for new agents. Team Owners can deny speci
The following tools can be individually toggled in the agent configuration dialog:
### 1. Sandboxed Terminal (`bash`)
- **Description:** Allows the agent to run terminal/shell commands in a highly isolated environment.
- **Why it matters:** Essential for running custom scripts, processing files on disk, or using standard Unix utilities. If disabled, the agent won't be able to run command-line code.
### 2. Read Skills (`read_skill`)
- **Description:** Allows the agent to load and read skill definitions to perform specialized actions.
- **Why it matters:** Enables the agent to inspect the instructions/configurations of custom skills registered in your workspace before executing them.
### 3. Analyze Image (`analyze_image`)
- **Description:** Allows the agent to view and analyze the visual contents of an image file.
- **Why it matters:** Enables multimodal agents to "see" image assets uploaded to Shumai, allowing them to answer comments or write metadata about images.
### 4. Video Frame Extraction (`screenshot`)
- **Description:** Allows the agent to extract still image frames from video files for analysis.
- **Why it matters:** Enables the agent to perform frame-by-frame analysis on videos (e.g. for comments, annotation, or search).
### 5. Browse Shumai (`list_assets`)
- **Description:** Allows the agent to list folders and browse files in Shumai.
- **Why it matters:** Gives the agent visual mapping of the project structure so it can search for other files or locate related assets.
### 6. Create Folder (`create_folder`)
- **Description:** Allows the agent to create new folders in Shumai.
- **Why it matters:** Needed if the agent organizes outputs or places files in custom subdirectory folders.
### 7. Create File (`create_file`)
- **Description:** Allows the agent to create and upload new files to Shumai.
- **Why it matters:** Allows the agent to generate new assets (e.g. text logs, transcoded clips, reports) and save them directly back into your project.
### 8. Create Version (`create_version`)
- **Description:** Allows the agent to upload and stack a new version of an existing file in Shumai.
- **Why it matters:** Allows the agent to update files in a clean, non-destructive way by stacking revisions rather than creating duplicate files.
@@ -54,6 +65,7 @@ The following tools can be individually toggled in the agent configuration dialo
## Restricting Tools
To change which tools an agent can use:
1. Navigate to **Team Settings** > **Agents** (Note: Only Team Owners can view and modify these settings).
2. Click on the agent you want to edit.
3. Scroll down below the **Skills** section to find the **Tools** configuration panel.
+1
View File
@@ -76,6 +76,7 @@ export default tseslint.config(
'**/data/**',
'**/packages/webui/components/ui/**',
'**/packages/webui/paraglide/**',
'**/packages/webui/public/**',
],
},
js.configs.recommended,
+1 -5
View File
@@ -1,6 +1,6 @@
{
"name": "shumai",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"repository": {
@@ -66,9 +66,5 @@
"typescript-eslint": "8.60.1",
"vite-tsconfig-paths": "6.1.1",
"vitest": "4.1.8"
},
"dependencies": {
"@tanstack/react-virtual": "3.14.2",
"jittered-fractional-indexing": "1.0.1"
}
}
+3 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/agent",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -20,7 +20,8 @@
"@sinclair/typebox": "0.34.49",
"ulid": "3.0.2",
"adm-zip": "0.5.17",
"zod": "4.4.3"
"zod": "4.4.3",
"jittered-fractional-indexing": "1.0.1"
},
"devDependencies": {
"@types/adm-zip": "0.5.8"
@@ -596,6 +596,45 @@ describe('Agent Database Activities Integration', () => {
const fetchedComment = await getCommentActivity(comment.id)
expect(fetchedComment?.id).toBe(comment.id)
})
it('should resolve version stack asset to its latest version file', async () => {
const stack = await prisma.asset.create({
data: {
name: 'Stack',
type: AssetType.version_stack,
status: AssetStatus.uploaded,
projectId: project.id,
},
})
await prisma.asset.create({
data: {
name: 'v1.png',
type: AssetType.file,
mediaType: 'image/png',
status: AssetStatus.uploaded,
projectId: project.id,
parentId: stack.id,
sortIndex: 'a1',
},
})
const v2 = await prisma.asset.create({
data: {
name: 'v2.png',
type: AssetType.file,
mediaType: 'image/png',
status: AssetStatus.uploaded,
projectId: project.id,
parentId: stack.id,
sortIndex: 'a0',
},
})
const fetchedAsset = await getAssetActivity(stack.id)
expect(fetchedAsset?.id).toBe(v2.id)
expect(fetchedAsset?.name).toBe('v2.png')
})
})
describe('getAssetPathContextActivity', () => {
+25 -1
View File
@@ -763,7 +763,7 @@ export async function getAgentAutofillContextActivity(params: {
}
export async function getAssetActivity(assetId: string) {
return prisma.asset.findUnique({
const asset = await prisma.asset.findUnique({
where: { id: assetId },
include: {
storageKey: true,
@@ -774,6 +774,30 @@ export async function getAssetActivity(assetId: string) {
},
},
})
if (asset && asset.type === AssetType.version_stack) {
const latestVersion = await prisma.asset.findFirst({
where: { parentId: asset.id, isDeleted: false },
orderBy: { sortIndex: 'asc' },
include: {
storageKey: true,
project: {
include: {
team: true,
},
},
},
})
if (latestVersion) {
return {
...latestVersion,
project: latestVersion.project ?? asset.project,
}
}
}
return asset
}
export async function getCommentActivity(commentId: string) {
+3 -2
View File
@@ -88,8 +88,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
outputTokens: 0,
}
const isImage = asset.mediaType.startsWith('image/')
const isVideo = asset.mediaType.startsWith('video/')
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
if (!isImage && !isVideo) {
throw ApplicationFailure.create({
@@ -409,6 +409,8 @@ describe('DatabaseSessionStorage', () => {
name: 'test.png',
type: 'file',
mediaType: 'image/png',
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
media: { proxyType: 'image' } as any,
projectId: project.id,
status: 'uploaded',
},
@@ -443,9 +445,9 @@ describe('DatabaseSessionStorage', () => {
providers: [],
})
// 4. Assert that the tools were instantiated using the new comment ID, not the original one
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(asset.id, 'new-comment-456')
expect(createScreenshotToolSpy).not.toHaveBeenCalled()
// 4. Assert that the tools were instantiated using the new comment ID and user ID
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
})
it('should use the current comment ID for media tools in subsequent turns (video)', async () => {
@@ -461,6 +463,8 @@ describe('DatabaseSessionStorage', () => {
name: 'test.mp4',
type: 'file',
mediaType: 'video/mp4',
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
media: { proxyType: 'video' } as any,
projectId: project.id,
status: 'uploaded',
},
@@ -495,8 +499,8 @@ describe('DatabaseSessionStorage', () => {
providers: [],
})
// 4. Assert that the tools were instantiated using the new comment ID, not the original one
expect(createScreenshotToolSpy).toHaveBeenCalledWith(asset.id, 'new-comment-456')
expect(createAnalyzeImageToolSpy).not.toHaveBeenCalled()
// 4. Assert that the tools were instantiated using the new comment ID and user ID
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
})
})
+1
View File
@@ -43,6 +43,7 @@ describe('formatSkillsForPrompt', () => {
// Verify lines and XML elements
expect(result).toContain('<available_skills>')
expect(result).toContain('<skill>')
expect(result).toContain('<id>skill-1</id>')
expect(result).toContain('<name>Deploy &amp; Test</name>')
expect(result).toContain(
'<description>Deploys code &lt;deploy&gt; and &quot;runs&quot; tests.</description>',
+8 -14
View File
@@ -15,7 +15,7 @@ import { agentService } from '@shumai/core/src/agent/agent'
import { DatabaseSessionStorage } from './database-session-storage'
import { createAnalyzeImageTool } from './tools/analyze-image'
import { createScreenshotTool } from './tools/screenshot'
import { getSimpleMediaType } from './workflows/agent-chat-prompt-builder'
import { createReadPdfPagesTool } from './tools/read-pdf-pages'
import { createCreateFileTool } from './tools/create-file'
import { createCreateFolderTool } from './tools/create-folder'
import { createCreateVersionTool } from './tools/create-version'
@@ -225,23 +225,16 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
}
const metadata = await storage.getMetadata()
const assetId = metadata.assetId
const userCommentId =
passedUserCommentId !== undefined ? passedUserCommentId : metadata.userCommentId
const mediaTools: AgentTool[] = []
if (assetId) {
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
if (asset?.mediaType) {
const type = getSimpleMediaType(asset.mediaType)
if (type === 'image') {
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
} else if (type === 'video') {
mediaTools.push(createScreenshotTool(assetId, userCommentId))
}
}
if (userId) {
mediaTools.push(
createAnalyzeImageTool(userId, userCommentId),
createScreenshotTool(userId, userCommentId),
createReadPdfPagesTool(userId, userCommentId),
)
}
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs, {
@@ -371,6 +364,7 @@ export function formatSkillsForPrompt(
for (const skill of skills) {
const filePath = path.join(process.cwd(), '.pi', 'skills', skill.id, 'SKILL.md')
lines.push(' <skill>')
lines.push(` <id>${escapeXml(skill.id)}</id>`)
lines.push(` <name>${escapeXml(skill.name)}</name>`)
lines.push(` <description>${escapeXml(skill.description || 'No description')}</description>`)
lines.push(` <location>${escapeXml(filePath)}</location>`)
+31 -6
View File
@@ -5,16 +5,21 @@ import {
WorkflowTaskStatus,
type Asset,
type AssetComment,
type User,
type WorkflowTask,
} from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
return {
...actual,
prisma: {
user: {
findUnique: vi.fn(),
},
asset: {
findUnique: vi.fn(),
},
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
},
}))
vi.mock('@shumai/core/src/authz/authz', () => ({
authzService: {
hasPermission: vi.fn(),
},
Permission: { Read: 'Read' },
ResourceType: { Asset: 'asset' },
}))
describe('analyzeImageTool', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('should fetch the image from S3 when comment has no annotations', async () => {
it('should check user permissions and fetch the image from S3 when comment has no annotations', async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -55,6 +71,7 @@ describe('analyzeImageTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
annotation: null,
} as unknown as AssetComment)
@@ -63,8 +80,15 @@ describe('analyzeImageTool', () => {
contentType: 'image/png',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createAnalyzeImageTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', {})
const tool = createAnalyzeImageTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1' })
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'asset-1',
})
expect(result.content[0].type).toBe('image')
expect((result.content[0] as unknown as { data: string }).data).toBe(
@@ -87,6 +111,7 @@ describe('analyzeImageTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
annotation: [
{
type: 'box',
@@ -114,14 +139,14 @@ describe('analyzeImageTool', () => {
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createAnalyzeImageTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', {})
const tool = createAnalyzeImageTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1' })
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: {
assetId: 'asset-1',
projectId: 'project-1',
type: 'transcode',
type: 'transcode_image_annotation',
status: 'pending',
payload: {
projectId: 'project-1',
+28 -7
View File
@@ -1,14 +1,19 @@
import { Type } from '@sinclair/typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { type ImageContent } from '@earendil-works/pi-ai'
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
const analyzeImageSchema = Type.Object({})
const analyzeImageSchema = Type.Object({
assetId: Type.String({
description: 'The asset ID of the image to analyze. This parameter is required.',
}),
})
export function createAnalyzeImageTool(
assetId: string,
userId: string,
userCommentId?: string | null,
): AgentTool<typeof analyzeImageSchema> {
return {
@@ -17,8 +22,24 @@ export function createAnalyzeImageTool(
description:
'Retrieves and views the image content. Call this tool if you need to analyze the image.',
parameters: analyzeImageSchema,
execute: async () => {
execute: async (_toolCallId, params) => {
try {
const assetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
include: { storageKey: true },
@@ -48,13 +69,13 @@ export function createAnalyzeImageTool(
}
}
// Check if trigger comment has draw annotations
// Check if trigger comment has draw annotations for this asset
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment?.annotation) {
if (comment && comment.assetId === assetId && comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
@@ -70,7 +91,7 @@ export function createAnalyzeImageTool(
data: {
assetId,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode,
type: WorkflowTaskType.transcode_image_annotation,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
+3
View File
@@ -34,6 +34,9 @@ function getMimeType(filePath: string): string {
'.mp4': 'video/mp4',
'.mov': 'video/quicktime',
'.pdf': 'application/pdf',
'.txt': 'text/plain',
'.md': 'text/markdown',
'.markdown': 'text/markdown',
}
return mimeTypes[ext] || 'application/octet-stream'
}
@@ -34,6 +34,9 @@ function getMimeType(filePath: string): string {
'.mp4': 'video/mp4',
'.mov': 'video/quicktime',
'.pdf': 'application/pdf',
'.txt': 'text/plain',
'.md': 'text/markdown',
'.markdown': 'text/markdown',
}
return mimeTypes[ext] || 'application/octet-stream'
}
@@ -0,0 +1,227 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { createReadPdfPagesTool } from './read-pdf-pages'
import {
prisma,
WorkflowTaskStatus,
type Asset,
type AssetComment,
type User,
type WorkflowTask,
} from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
return {
...actual,
prisma: {
user: {
findUnique: vi.fn(),
},
asset: {
findUnique: vi.fn(),
},
assetComment: {
findUnique: vi.fn(),
},
workflowTask: {
create: vi.fn(),
},
},
}
})
vi.mock('@shumai/core/src/s3/s3', () => ({
s3Service: {
getObject: vi.fn(),
},
}))
vi.mock('@shumai/workflow-core', () => ({
workflowService: {
executeWait: vi.fn(),
},
}))
vi.mock('@shumai/core/src/authz/authz', () => ({
authzService: {
hasPermission: vi.fn(),
},
Permission: { Read: 'Read' },
ResourceType: { Asset: 'asset' },
}))
describe('readPdfPagesTool', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(authzService.hasPermission).mockResolvedValue()
})
it('should trigger pdfPages transcode workflow and return image outputs', async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
media: { proxyType: 'pdf' },
} as unknown as Asset)
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
second: 2,
annotation: [
{
type: 'box',
color: '#ff0000',
points: [
[0, 0],
[1, 1],
],
},
],
} as unknown as AssetComment)
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
id: 'task-1',
} as unknown as WorkflowTask)
vi.mocked(workflowService.executeWait).mockResolvedValue({
id: 'task-1',
status: WorkflowTaskStatus.completed,
output: {
pages: [
{ key: 'pdf_pages/page_1.webp', page: 1 },
{ key: 'pdf_pages/page_2.webp', page: 2 },
],
},
} as unknown as WorkflowTask)
vi.mocked(s3Service.getObject).mockImplementation(async (_bucket: string, key: string) => {
return {
buffer: Buffer.from(`fake-bytes-for-${key}`),
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string }
})
const tool = createReadPdfPagesTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 2 })
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'asset-1',
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: {
assetId: 'asset-1',
projectId: 'project-1',
type: 'transcode_pdf_pages',
status: 'pending',
payload: {
projectId: 'project-1',
pdfPages: {
start: 1,
end: 2,
commentTimestamp: 2,
annotations: [
{
type: 'box',
color: '#ff0000',
points: [
[0, 0],
[1, 1],
],
},
],
},
},
},
})
expect(result.content.length).toBe(2)
expect(result.content[0].type).toBe('image')
expect((result.content[0] as unknown as { data: string }).data).toBe(
Buffer.from('fake-bytes-for-pdf_pages/page_1.webp').toString('base64'),
)
expect(result.details.sourceKeys).toEqual(['pdf_pages/page_1.webp', 'pdf_pages/page_2.webp'])
})
it('should return error text if start page is less than 1', async () => {
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Invalid page range: start page (0) must be at least 1.',
})
})
it('should return error text if start page is greater than end page', async () => {
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Invalid page range: start page (5) must be less than or equal to end page (2).',
})
})
it('should return error text if page range exceeds maximum limit of 20', async () => {
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
})
})
it('should return error text if asset not found', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Asset with ID asset-1 not found.',
})
})
it('should return error text if proxyType is not pdf', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
media: { proxyType: 'image' },
} as unknown as Asset)
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Asset asset-1 is not a PDF or document.',
})
})
it('should return error text if proxyType is null/undefined (unprocessed)', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
media: null,
} as unknown as Asset)
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Asset asset-1 is not a PDF or document.',
})
})
})
+186
View File
@@ -0,0 +1,186 @@
import { Type } from '@sinclair/typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { type ImageContent } from '@earendil-works/pi-ai'
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
const readPdfPagesSchema = Type.Object({
assetId: Type.String({
description: 'Asset ID of the PDF document. This parameter is required.',
}),
start: Type.Number({ description: 'Start page number (1-based index)' }),
end: Type.Number({ description: 'End page number (1-based index)' }),
})
export function createReadPdfPagesTool(
userId: string,
userCommentId?: string | null,
): AgentTool<typeof readPdfPagesSchema> {
return {
name: 'read_pdf_pages',
label: 'Read PDF Pages',
description:
'Converts specific page range of a PDF document into 1080p WebP images for visual analysis. Maximum 20 pages allowed per call.',
parameters: readPdfPagesSchema,
execute: async (_toolCallId, params) => {
try {
if (params.start < 1) {
return {
content: [
{
type: 'text',
text: `Invalid page range: start page (${params.start}) must be at least 1.`,
},
],
details: {},
}
}
if (params.start > params.end) {
return {
content: [
{
type: 'text',
text: `Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
},
],
details: {},
}
}
const MAX_PAGE_RANGE = 20
const requestedPages = params.end - params.start + 1
if (requestedPages > MAX_PAGE_RANGE) {
return {
content: [
{
type: 'text',
text: `Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
},
],
details: {},
}
}
const targetAssetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: targetAssetId,
})
const asset = await prisma.asset.findUnique({
where: { id: targetAssetId },
})
if (!asset) {
return {
content: [{ type: 'text', text: `Asset with ID ${targetAssetId} not found.` }],
details: {},
}
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
if (proxyType !== 'pdf') {
return {
content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }],
details: {},
}
}
// Fetch trigger comment details for comment page and annotations
let commentTimestamp: number | null = null
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment && comment.assetId === targetAssetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
}
}
}
// Trigger transcode workflow to render PDF pages
const task = await prisma.workflowTask.create({
data: {
assetId: targetAssetId,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_pdf_pages,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
pdfPages: {
start: params.start,
end: params.end,
commentTimestamp,
annotations: annotations as PrismaJson.AnnotationList,
},
},
},
})
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
if (!pages || pages.length === 0) {
return {
content: [{ type: 'text', text: 'No PDF page images were generated.' }],
details: {},
}
}
const content: ImageContent[] = []
const sourceKeys: string[] = []
const bucket = process.env.S3_BUCKET || 'shumai'
for (const pageItem of pages) {
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
})
sourceKeys.push(pageItem.key)
}
return {
content,
details: {
sourceKeys,
},
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error reading PDF pages: ${
error instanceof Error ? error.message : String(error)
}`,
},
],
details: {},
}
}
},
}
}
+27 -3
View File
@@ -5,16 +5,21 @@ import {
WorkflowTaskStatus,
type Asset,
type AssetComment,
type User,
type WorkflowTask,
} from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
return {
...actual,
prisma: {
user: {
findUnique: vi.fn(),
},
asset: {
findUnique: vi.fn(),
},
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
},
}))
vi.mock('@shumai/core/src/authz/authz', () => ({
authzService: {
hasPermission: vi.fn(),
},
Permission: { Read: 'Read' },
ResourceType: { Asset: 'asset' },
}))
describe('screenshotTool', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('should trigger screenshot transcode workflow and return image outputs', async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -53,6 +69,7 @@ describe('screenshotTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
second: 5.0,
annotation: [
{
@@ -88,14 +105,21 @@ describe('screenshotTool', () => {
} as unknown as { buffer: Buffer; contentType: string }
})
const tool = createScreenshotTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', { start: 0, end: 10, count: 2 })
const tool = createScreenshotTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 10, count: 2 })
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'asset-1',
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: {
assetId: 'asset-1',
projectId: 'project-1',
type: 'transcode',
type: 'transcode_screenshot',
status: 'pending',
payload: {
projectId: 'project-1',
+24 -4
View File
@@ -1,18 +1,22 @@
import { Type } from '@sinclair/typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { type ImageContent } from '@earendil-works/pi-ai'
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
const screenshotSchema = Type.Object({
assetId: Type.String({
description: 'The asset ID of the video asset. This parameter is required.',
}),
start: Type.Number({ description: 'Start time in seconds' }),
end: Type.Number({ description: 'End time in seconds' }),
count: Type.Number({ description: 'Number of screenshots to take' }),
})
export function createScreenshotTool(
assetId: string,
userId: string,
userCommentId?: string | null,
): AgentTool<typeof screenshotSchema> {
return {
@@ -23,6 +27,22 @@ export function createScreenshotTool(
parameters: screenshotSchema,
execute: async (_toolCallId, params) => {
try {
const assetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
@@ -41,7 +61,7 @@ export function createScreenshotTool(
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment) {
if (comment && comment.assetId === assetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
@@ -57,7 +77,7 @@ export function createScreenshotTool(
data: {
assetId,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode,
type: WorkflowTaskType.transcode_screenshot,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
@@ -64,6 +64,7 @@ describe('Agent Autofill Workflow', () => {
storageKey: { key: 'asset-key' },
project: { id: 'p1', teamId: 't1' },
mediaType: 'image/png',
media: { proxyType: 'image' },
})
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/test.png',
@@ -290,6 +291,7 @@ describe('Agent Autofill Workflow', () => {
storageKey: { key: 'asset-key' },
project: { id: 'p1', teamId: 't1' },
mediaType: 'audio/mp3',
media: { proxyType: 'audio' },
})
const task = await prisma.workflowTask.create({
@@ -64,8 +64,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
const projectId = asset.project.id
// 2. Prepare Data (Images)
const isImage = asset.mediaType?.startsWith('image/') || false
const isVideo = asset.mediaType?.startsWith('video/') || false
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
if (!isImage && !isVideo) {
throw ApplicationFailure.create({
@@ -25,7 +25,13 @@ describe('AgentChatPromptBuilder', () => {
})
it('should build prompt with image asset details', () => {
const builder = new AgentChatPromptBuilder('a1').withAssetDetails('test.png', 'image/png')
const builder = new AgentChatPromptBuilder('a1').withAssetDetails(
'test.png',
'image/png',
undefined,
undefined,
'image',
)
const result = builder.build()
expect(result).toContain('File Name: test.png')
@@ -39,6 +45,8 @@ describe('AgentChatPromptBuilder', () => {
'movie.mp4',
'video/mp4',
12.34,
undefined,
'video',
)
const result = builder.build()
@@ -49,6 +57,45 @@ describe('AgentChatPromptBuilder', () => {
expect(result).not.toContain("call the 'analyze_image' tool")
})
it('should build prompt with PDF asset details and total pages', () => {
const builder = new AgentChatPromptBuilder('a1').withAssetDetails(
'doc.pdf',
'application/pdf',
undefined,
15,
'pdf',
)
const result = builder.build()
expect(result).toContain('File Name: doc.pdf')
expect(result).toContain('File Type: pdf')
expect(result).toContain('Total Pages: 15')
expect(result).toContain("call the 'read_pdf_pages' tool")
expect(result).toContain('Maximum 20 pages allowed per call.')
expect(result).not.toContain("call the 'screenshot' tool")
expect(result).not.toContain("call the 'analyze_image' tool")
})
it('should build prompt with comment page for PDF asset', () => {
const builder = new AgentChatPromptBuilder('a1')
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
.withCommentTimestamp(3)
const result = builder.build()
expect(result).toContain('Comment Page: 3')
})
it('should build prompt for PDF continuation mode with comment page', () => {
const builder = new AgentChatPromptBuilder('a1')
.withContinuation(true)
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
.withCommentTimestamp(4)
const result = builder.build()
expect(result).toContain('Comment Page: 4')
expect(result).not.toContain('Comment Timestamp:')
})
it('should build prompt with comment timestamp', () => {
const builder = new AgentChatPromptBuilder('a1').withCommentTimestamp(5.5)
const result = builder.build()
@@ -60,7 +107,7 @@ describe('AgentChatPromptBuilder', () => {
const builder = new AgentChatPromptBuilder('a1')
.withExplicitMention(true)
.withPathContext('src/main.ts')
.withAssetDetails('movie.mp4', 'video/mp4', 12.34)
.withAssetDetails('movie.mp4', 'video/mp4', 12.34, undefined, 'video')
.withCommentTimestamp(8.2)
const result = builder.build()
@@ -103,4 +150,18 @@ describe('AgentChatPromptBuilder', () => {
expect(result).toContain('Attached Files:\n- file1.txt')
expect(result).toContain('Referenced Workspace Assets:\n- folder1')
})
it('should include location change context block in continuation mode when asset has changed', () => {
const builder = new AgentChatPromptBuilder('a2')
.withContinuation(true)
.withAssetChanged(true)
.withAssetDetails('new-folder', 'folder', undefined, undefined, 'folder')
.withPathContext('projects/my-proj/new-folder')
const result = builder.build()
expect(result).toContain('[Context: User Navigated to a New Location]')
expect(result).toContain('New Location Asset ID: a2')
expect(result).toContain('File Name: new-folder')
expect(result).toContain('Asset Path Context:\nprojects/my-proj/new-folder')
})
})
@@ -1,16 +1,7 @@
export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'other' {
if (!mediaType) return 'other'
const lower = mediaType.toLowerCase()
if (
lower.startsWith('image/') ||
lower === 'application/x-photoshop' ||
lower === 'image/vnd.adobe.photoshop'
) {
return 'image'
}
if (lower.startsWith('video/')) {
return 'video'
}
export function getSimpleMediaType(proxyType?: string | null): 'image' | 'video' | 'pdf' | 'other' {
if (proxyType === 'image') return 'image'
if (proxyType === 'video') return 'video'
if (proxyType === 'pdf') return 'pdf'
return 'other'
}
@@ -18,7 +9,9 @@ export class AgentChatPromptBuilder {
private assetId: string
private assetName?: string
private mediaType?: string
private proxyType?: string
private videoDuration?: number
private totalPages?: number
private commentTimestamp?: number
private pathContext?: string
private explicitMention = false
@@ -26,6 +19,7 @@ export class AgentChatPromptBuilder {
private referencedAssets: string[] = []
private isContinuation = false
private hasAssetChanged = false
constructor(assetId: string) {
this.assetId = assetId
@@ -36,6 +30,11 @@ export class AgentChatPromptBuilder {
return this
}
withAssetChanged(hasAssetChanged?: boolean): this {
this.hasAssetChanged = !!hasAssetChanged
return this
}
withAttachedFiles(files?: string[]): this {
if (files) {
this.attachedFiles = files
@@ -55,10 +54,18 @@ export class AgentChatPromptBuilder {
return this
}
withAssetDetails(name: string, mediaType: string | null, duration?: number): this {
withAssetDetails(
name: string,
mediaType: string | null,
duration?: number,
totalPages?: number,
proxyType?: string | null,
): this {
this.assetName = name
this.mediaType = mediaType || undefined
this.proxyType = proxyType || undefined
this.videoDuration = duration
this.totalPages = totalPages
return this
}
@@ -75,13 +82,40 @@ export class AgentChatPromptBuilder {
}
build(): string {
const effectiveType = this.proxyType
if (this.isContinuation) {
let instruction = ''
if (this.hasAssetChanged) {
instruction += `[Context: User Navigated to a New Location]\nNew Location Asset ID: ${this.assetId}`
if (this.assetName) {
instruction += `\nFile Name: ${this.assetName}`
}
if (effectiveType) {
const type = getSimpleMediaType(effectiveType)
instruction += `\nFile Type: ${type}`
if (type === 'video' && this.videoDuration !== undefined) {
instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds`
} else if (type === 'pdf' && this.totalPages !== undefined) {
instruction += `\nTotal Pages: ${this.totalPages}`
}
}
if (this.pathContext) {
instruction += `\n\nAsset Path Context:\n${this.pathContext}`
}
}
if (this.commentTimestamp !== undefined) {
instruction += `Comment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
const type = getSimpleMediaType(effectiveType)
if (instruction) instruction += '\n\n'
if (type === 'pdf') {
instruction += `Comment Page: ${Math.round(this.commentTimestamp)}`
} else {
instruction += `Comment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
}
}
if (this.attachedFiles.length > 0 || this.referencedAssets.length > 0) {
instruction += `\n\n[Context: New Attached Files & Referenced Assets]`
if (instruction) instruction += '\n\n'
instruction += `[Context: New Attached Files & Referenced Assets]`
if (this.attachedFiles.length > 0) {
instruction += `\nAttached Files:\n${this.attachedFiles.join('\n')}`
}
@@ -96,15 +130,22 @@ export class AgentChatPromptBuilder {
if (this.assetName) {
instruction += `\nFile Name: ${this.assetName}`
}
if (this.mediaType) {
const type = getSimpleMediaType(this.mediaType)
if (effectiveType) {
const type = getSimpleMediaType(effectiveType)
instruction += `\nFile Type: ${type}`
if (type === 'video' && this.videoDuration !== undefined) {
instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds`
} else if (type === 'pdf' && this.totalPages !== undefined) {
instruction += `\nTotal Pages: ${this.totalPages}`
}
}
if (this.commentTimestamp !== undefined) {
instruction += `\nComment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
const type = getSimpleMediaType(effectiveType)
if (type === 'pdf') {
instruction += `\nComment Page: ${Math.round(this.commentTimestamp)}`
} else {
instruction += `\nComment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
}
}
if (this.pathContext) {
instruction += `\n\nAsset Path Context:\n${this.pathContext}`
@@ -120,12 +161,14 @@ export class AgentChatPromptBuilder {
}
}
if (this.mediaType) {
const type = getSimpleMediaType(this.mediaType)
if (effectiveType) {
const type = getSimpleMediaType(effectiveType)
if (type === 'image') {
instruction += `\n\nIf you need to view the image data, call the 'analyze_image' tool. It does not require any parameters.`
} else if (type === 'video') {
instruction += `\n\nIf you need to view visual frames or take screenshots of the video, call the 'screenshot' tool. You must specify the 'start' (seconds), 'end' (seconds), and 'count' (number of screenshots) parameters.`
} else if (type === 'pdf') {
instruction += `\n\nIf you need to view pages of the PDF document, call the 'read_pdf_pages' tool. You must specify the 'start' (page number) and 'end' (page number) parameters. Maximum 20 pages allowed per call.`
}
}
@@ -61,6 +61,7 @@ describe('Agent Chat Workflow', () => {
type: 'file',
name: 'test-file.png',
mediaType: 'image/png',
media: { proxyType: 'image' },
parentId: 'parent-folder-id',
})
mockActivities.getCommentActivity.mockResolvedValue({
@@ -133,7 +134,7 @@ describe('Agent Chat Workflow', () => {
// Verify agent instruction composition
const expectedInstruction1 = new AgentChatPromptBuilder('a1')
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('test-file.png', 'image/png', undefined)
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
.withCommentTimestamp(null)
.withExplicitMention(true)
.build()
@@ -239,7 +240,7 @@ describe('Agent Chat Workflow', () => {
// Verify instruction contains not explicitly mentioned instructions
const expectedInstruction3 = new AgentChatPromptBuilder('a1')
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('test-file.png', 'image/png', undefined)
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
.withCommentTimestamp(null)
.withExplicitMention(false)
.build()
@@ -271,18 +272,21 @@ describe('Agent Chat Workflow', () => {
{
asset: {
mediaType: 'image/png',
media: { proxyType: 'image' },
storageKey: { key: 'attachments/image1.png' },
},
},
{
asset: {
mediaType: 'video/mp4', // non-image, should be ignored
media: { proxyType: 'video' },
storageKey: { key: 'attachments/video.mp4' },
},
},
{
asset: {
mediaType: 'image/jpeg',
media: { proxyType: 'image' },
storageKey: { key: 'attachments/image2.jpg' },
},
},
@@ -305,7 +309,7 @@ describe('Agent Chat Workflow', () => {
const expectedInstruction4 = new AgentChatPromptBuilder('a1')
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('test-file.png', 'image/png', undefined)
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
.withCommentTimestamp(null)
.withExplicitMention(false)
.build()
@@ -512,4 +516,55 @@ describe('Agent Chat Workflow', () => {
}),
)
})
it('should handle version stack asset by building instructions with resolved latest version asset details', async () => {
mockActivities.getAssetActivity.mockImplementation(async (id: string) => {
if (id === 'vs-1') {
return {
id: 'v2-latest',
project: { teamId: 't1' },
type: 'file',
name: 'latest-video.mp4',
mediaType: 'video/mp4',
media: { proxyType: 'video', duration: 42 },
parentId: 'vs-1',
}
}
return null
})
const task = await prisma.workflowTask.create({
data: {
type: 'chat',
status: 'pending',
assetId: 'vs-1',
payload: {
projectId: 'p1',
agent: {
prompt: 'Explain this version stack video',
sessionId: 'session-vs-123',
agentId: 'b1',
},
},
},
})
await agentChat(task)
const expectedInstruction = new AgentChatPromptBuilder('v2-latest')
.withContinuation(true)
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('latest-video.mp4', 'video/mp4', 42, undefined, 'video')
.build()
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
expect.objectContaining({
teamId: 't1',
agentId: 'b1',
message: 'Explain this version stack video',
agentsInstruction: expectedInstruction,
sessionId: 'session-vs-123',
}),
)
})
})
+6 -2
View File
@@ -74,7 +74,8 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
// 5. Prepare Images (Attachments only)
if (userComment.attachments) {
for (const att of userComment.attachments) {
if (att.asset?.mediaType?.startsWith('image/') && att.asset?.storageKey?.key) {
const attProxyType = (att.asset?.media as PrismaJson.MediaInfo | null)?.proxyType
if (attProxyType === 'image' && att.asset?.storageKey?.key) {
attachmentImageUrls.push(att.asset.storageKey.key)
}
}
@@ -124,6 +125,7 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const mediaInfo = asset.media as any
const duration = mediaInfo?.duration
const totalPages = mediaInfo?.frames || mediaInfo?.metadata?.totalFrames
// Resolve Attached Files Details
const attachedFileDetailsList: string[] = []
@@ -160,10 +162,12 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
}
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const instruction = new AgentChatPromptBuilder(asset.id)
.withContinuation(!isNewChat)
.withAssetChanged(payload.agent?.hasAssetChanged)
.withPathContext(pathContext)
.withAssetDetails(asset.name, asset.mediaType, duration)
.withAssetDetails(asset.name, asset.mediaType, duration, totalPages, proxyType)
.withCommentTimestamp(commentTimestamp)
.withExplicitMention(payload.agent?.explicitMention)
.withAttachedFiles(attachedFileDetailsList)
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/api",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
+3 -3
View File
@@ -79,7 +79,7 @@ describe('file api', () => {
fileCount: 1,
type: 'file',
status: 'active',
mediaType: 'image/jpeg',
proxyType: 'image',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@@ -107,7 +107,7 @@ describe('file api', () => {
fileCount: 1,
type: 'file',
status: 'active',
mediaType: 'image/jpeg',
proxyType: 'image',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@@ -500,7 +500,7 @@ describe('file api', () => {
fileCount: 1,
type: 'file',
status: 'active',
mediaType: 'image/jpeg',
proxyType: 'image',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
-6
View File
@@ -53,7 +53,6 @@ describe('folder api', () => {
fileCount: 1,
type: 'folder',
status: 'active',
mediaType: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@@ -81,7 +80,6 @@ describe('folder api', () => {
fileCount: 1,
type: 'folder',
status: 'active',
mediaType: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@@ -113,7 +111,6 @@ describe('folder api', () => {
fileCount: 0,
type: 'folder',
status: 'active',
mediaType: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
@@ -147,7 +144,6 @@ describe('folder api', () => {
fileCount: 0,
type: 'folder',
status: 'active',
mediaType: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
},
@@ -223,7 +219,6 @@ describe('folder api', () => {
fileCount: 0,
type: 'folder',
status: 'active',
mediaType: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
},
@@ -261,7 +256,6 @@ describe('folder api', () => {
fileCount: 1,
type: 'folder',
status: 'active',
mediaType: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
+6 -3
View File
@@ -1,23 +1,25 @@
{
"name": "@shumai/core",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@earendil-works/pi-agent-core": "0.78.0",
"@sinclair/typebox": "0.34.49",
"adm-zip": "0.5.17",
"better-auth": "1.6.11",
"csv-parse": "7.0.1",
"date-fns": "4.4.0",
"dotenv": "16.6.1",
"dotenv-expand": "13.0.0",
"extract-zip": "2.0.1",
"hono": "4.12.23",
"jittered-fractional-indexing": "1.0.1",
"pdfkit": "0.19.1",
"pino": "10.3.1",
"sharp": "0.34.5",
"tweetnacl": "1.0.3",
@@ -27,6 +29,7 @@
},
"devDependencies": {
"@types/adm-zip": "0.5.8",
"@types/extract-zip": "2.0.3"
"@types/extract-zip": "2.0.3",
"@types/pdfkit": "0.17.6"
}
}
+5 -2
View File
@@ -1475,6 +1475,7 @@ describe('AssetService', () => {
mediaType: 'video/mp4',
status: 'processed',
media: {
proxyType: 'video',
poster: { key: 'poster-key' },
metadata: { duration: 262 },
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -1485,7 +1486,7 @@ describe('AssetService', () => {
const info = await assetService.getAsset({ assetId: video.id })
expect(info.preview).toBeDefined()
expect(info.preview?.mediaType).toBe('video/mp4')
expect(info.preview?.proxyType).toBe('video')
expect(info.preview?.duration).toBe(262)
})
@@ -1502,6 +1503,7 @@ describe('AssetService', () => {
mediaType: 'image/png',
status: 'processed',
media: {
proxyType: 'image',
thumbnail: { key: 'thumb-key' },
metadata: { originalWidth: 100, originalHeight: 80 },
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -1530,6 +1532,7 @@ describe('AssetService', () => {
mediaType: 'image/png',
status: 'processed',
media: {
proxyType: 'image',
thumbnail: { key: 'thumb-key' },
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any,
@@ -1554,7 +1557,7 @@ describe('AssetService', () => {
expect(info.type).toBe(AssetType.symlink)
// Resolved from target
expect(info.sizeByte).toBe(5000)
expect(info.mediaType).toBe('image/png')
expect(info.proxyType).toBe('image')
expect(info.media).toBeDefined()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((info.media as any).thumbnail?.key).toBe('thumb-key')
+34 -5
View File
@@ -102,6 +102,7 @@ export class AssetService {
metadataValues: true,
storageKey: true,
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -109,6 +110,7 @@ export class AssetService {
},
},
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -661,6 +663,7 @@ export class AssetService {
metadataValues: true,
storageKey: true,
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -668,6 +671,7 @@ export class AssetService {
},
},
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -745,6 +749,7 @@ export class AssetService {
metadataValues: true,
storageKey: true,
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -752,6 +757,7 @@ export class AssetService {
},
},
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -794,6 +800,7 @@ export class AssetService {
metadataValues: true,
storageKey: true,
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -801,6 +808,7 @@ export class AssetService {
},
},
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -936,6 +944,7 @@ export class AssetService {
metadataValues: true,
storageKey: true,
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -943,6 +952,7 @@ export class AssetService {
},
},
children: {
where: { isDeleted: false },
include: { creator: true, metadataValues: true, storageKey: true },
take: 3,
orderBy: { sortIndex: 'asc' },
@@ -1641,6 +1651,7 @@ export class AssetService {
folderForPreview.children
) {
for (const child of folderForPreview.children) {
if (child.isDeleted) continue
let previewAsset = child as Asset
let mediaType = child.mediaType
@@ -1697,6 +1708,13 @@ export class AssetService {
'GET',
)
}
if (media && media.pdfTranscode?.key) {
media.pdfTranscode.url = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
media.pdfTranscode.key,
'GET',
)
}
const key = latestVersion.storageKey?.key
if (key) {
@@ -1715,6 +1733,8 @@ export class AssetService {
}
}
const proxyType = (media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
result.push({
id: a.id,
name:
@@ -1726,7 +1746,7 @@ export class AssetService {
type: a.type,
targetType: a.type === AssetType.symlink ? a.target?.type : null,
status: latestVersion.status,
mediaType: latestVersion.mediaType,
proxyType,
latestChildren,
preview,
createdAt: a.createdAt.toISOString(),
@@ -1990,14 +2010,16 @@ export class AssetService {
private async toPreviewInfo(asset: Asset | AssetWithIncludes): Promise<PreviewInfo | null> {
if (!asset.media) return null
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
let thumbnailUrl = undefined
if (asset.mediaType?.startsWith('image/') && asset.media.thumbnail?.key) {
if (proxyType === 'image' && asset.media.thumbnail?.key) {
thumbnailUrl = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
asset.media.thumbnail.key,
'GET',
)
} else if (asset.mediaType?.startsWith('video/') && asset.media.poster?.key) {
} else if (asset.media.poster?.key) {
thumbnailUrl = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
asset.media.poster.key,
@@ -2015,12 +2037,13 @@ export class AssetService {
}
return {
mediaType: asset.mediaType,
proxyType,
thumbnailUrl,
originalHeight: asset.media.metadata?.originalHeight,
originalWidth: asset.media.metadata?.originalWidth,
spriteUrl,
duration: asset.media.metadata?.duration,
pageCount: asset.media.metadata?.totalFrames,
}
}
@@ -2043,11 +2066,17 @@ export class AssetService {
a.asset.storageKey.key,
'GET',
)
const attachmentProxyType = (a.asset.media?.proxyType || null) as
| 'image'
| 'video'
| 'audio'
| 'pdf'
| null
attachments.push({
id: a.id,
assetId: a.asset.id,
url,
mediaType: a.asset.mediaType,
proxyType: attachmentProxyType,
})
}
}
+47
View File
@@ -155,6 +155,51 @@ describe('ChatService', () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const taskPayload = task?.payload as any
expect(taskPayload?.agent?.prompt).toBe('second message')
expect(taskPayload?.agent?.hasAssetChanged).toBe(false)
})
it('should update session assetId and set hasAssetChanged when continuing chat with a new contextAssetId', async () => {
const { user, team, project, rootFolder } = await setupBasicData()
const newFolder = await prisma.asset.create({
data: {
name: 'new_folder',
type: 'folder',
status: 'processed',
projectId: project.id,
parentId: rootFolder.id,
},
})
// Start session on rootFolder
const { sessionId } = await chatService.startOrContinueChat(user, team.id, {
agentId: 'test-agent-id',
textPrompt: 'message in root folder',
projectId: project.id,
contextAssetId: rootFolder.id,
})
const sessionBefore = await prisma.agentSession.findUnique({ where: { id: sessionId } })
expect(sessionBefore?.assetId).toBe(rootFolder.id)
// Continue session on newFolder
const { taskId: nextTaskId } = await chatService.startOrContinueChat(user, team.id, {
agentId: 'test-agent-id',
textPrompt: 'message after switching to new folder',
sessionId,
contextAssetId: newFolder.id,
})
// Verify session assetId updated
const sessionAfter = await prisma.agentSession.findUnique({ where: { id: sessionId } })
expect(sessionAfter?.assetId).toBe(newFolder.id)
// Verify task payload has assetId set to newFolder.id and hasAssetChanged is true
const task = await prisma.workflowTask.findUnique({ where: { id: nextTaskId } })
expect(task?.assetId).toBe(newFolder.id)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const taskPayload = task?.payload as any
expect(taskPayload?.agent?.hasAssetChanged).toBe(true)
})
it('should inject context of referenced assets and attached files', async () => {
@@ -181,6 +226,8 @@ describe('ChatService', () => {
name: 'doc.jpg',
type: 'file',
mediaType: 'image/jpeg',
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
media: { proxyType: 'image' } as any,
status: 'processed',
projectId: project.id,
parentId: childFolder.id,
+21 -4
View File
@@ -185,6 +185,7 @@ export class ChatService {
// 2. Project / Asset scoping resolution
let targetAssetId = ''
let hasAssetChanged = false
if (passedSessionId) {
const session = await this.prismaClient.agentSession.findUnique({
@@ -194,7 +195,18 @@ export class ChatService {
if (!session) throw new Error('Session not found')
if (session.userId !== user.id) throw new Error('Unauthorized session access')
if (!session.assetId) throw new Error('Session has no associated asset')
targetAssetId = session.assetId
if (contextAssetId && contextAssetId !== session.assetId) {
const contextAsset = await this.prismaClient.asset.findUnique({
where: { id: contextAssetId },
include: { project: true },
})
if (!contextAsset) throw new Error('Context asset not found')
targetAssetId = contextAssetId
hasAssetChanged = true
} else {
targetAssetId = session.assetId
}
} else {
// Start a new session
if (contextAssetId) {
@@ -232,7 +244,8 @@ export class ChatService {
where: { id: fileId },
include: { storageKey: true },
})
if (file && file.mediaType?.startsWith('image/') && file.storageKey?.key) {
const isImage = (file?.media as PrismaJson.MediaInfo | null)?.proxyType === 'image'
if (file && isImage && file.storageKey?.key) {
resolvedImageUrls.push(file.storageKey.key)
}
}
@@ -268,10 +281,13 @@ export class ChatService {
if (!sessionExists) throw new Error('Session not found')
if (sessionExists.userId !== user.id) throw new Error('Unauthorized session access')
// Update the session's agentId to match the selected agent
// Update the session's agentId and assetId (if location changed)
await tx.agentSession.update({
where: { id: activeSessionId },
data: { agentId },
data: {
agentId,
...(hasAssetChanged ? { assetId: targetAssetId } : {}),
},
})
} else {
const newSession = await tx.agentSession.create({
@@ -306,6 +322,7 @@ export class ChatService {
sessionId: activeSessionId,
userId: user.id,
isNewChat: !passedSessionId,
hasAssetChanged,
},
},
},
@@ -112,6 +112,23 @@ export const systemFields: Prisma.MetadataFieldCreateInput[] = [
},
},
},
{
key: 'proxy_type',
scope: 'SYSTEM',
readOnly: true,
config: {
name: 'Proxy Type',
type: 'select',
select: {
options: [
{ id: 'image', displayName: 'Image', color: 'system' },
{ id: 'video', displayName: 'Video', color: 'system' },
{ id: 'audio', displayName: 'Audio', color: 'system' },
{ id: 'pdf', displayName: 'PDF', color: 'system' },
],
},
},
},
{
key: 'audio_bit_depth',
scope: 'SYSTEM',
@@ -401,7 +401,7 @@ export class NotificationService {
info.asset = {
id: a.id,
name: a.name,
mediaType: a.mediaType || undefined,
proxyType: (a.media as PrismaJson.MediaInfo | null)?.proxyType || undefined,
preview: previewUrl || undefined,
thumbnailUrl: thumbnailUrl || undefined,
originalWidth: originalWidth || undefined,
+52
View File
@@ -7,6 +7,7 @@ import { metadataService } from '@shumai/core/src/metadata/metadata'
import { workflowService } from '@shumai/workflow-core'
import { AssetType } from '@shumai/db'
import { assetService } from '@shumai/core/src/asset/asset'
describe('SearchService', () => {
setupTestDbHooks()
@@ -559,6 +560,57 @@ describe('SearchService', () => {
expect(resultWithSymlink.data.find((a) => a.id === symlink.id)).toBeDefined()
})
it('excludes soft-deleted (trashed) files from latestChildren when searching folders', async () => {
const { rootFolder } = await setupBasicAssets()
const subfolder = await prisma.asset.create({
data: {
name: 'subfolder',
type: AssetType.folder,
projectId: rootFolder.projectId,
parentId: rootFolder.id,
status: 'uploaded',
},
})
const childFile1 = await prisma.asset.create({
data: {
name: 'child1.txt',
type: AssetType.file,
projectId: rootFolder.projectId,
parentId: subfolder.id,
status: 'uploaded',
sortIndex: 'a0',
},
})
await prisma.asset.create({
data: {
name: 'child2.txt',
type: AssetType.file,
projectId: rootFolder.projectId,
parentId: subfolder.id,
status: 'uploaded',
sortIndex: 'a1',
},
})
await assetService.deleteAssets([childFile1.id])
const result = await searchService.search(rootFolder.id, {
recursively: false,
assetType: 'folder',
operator: 'AND',
conditions: [],
isSemantic: false,
})
const searchedSubfolder = result.data.find((a) => a.id === subfolder.id)
expect(searchedSubfolder).toBeDefined()
expect(searchedSubfolder?.latestChildren?.length).toBe(1)
})
describe('Semantic Search with multiple chunks and distance ordering', () => {
it('returns multiple matching segments from the same asset and orders by distance', async () => {
const user = await prisma.user.create({
+280 -2
View File
@@ -1,11 +1,20 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { transcodeService } from './transcode'
import { s3Service } from '@shumai/core/src/s3/s3'
import * as path from 'path'
import * as child_process from 'child_process'
import * as fs from 'fs'
import { WorkflowTask } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import sharp from 'sharp'
vi.mock('@shumai/core/src/s3/s3', () => ({
s3Service: {
downloadToFile: vi.fn(),
putObject: vi.fn(),
},
}))
// Mock child_process
vi.mock('child_process', () => ({
execFile: vi.fn(),
@@ -16,7 +25,12 @@ vi.mock('sharp', () => {
const mockSharp = {
resize: vi.fn().mockReturnThis(),
webp: vi.fn().mockReturnThis(),
toFile: vi.fn().mockResolvedValue({}),
composite: vi.fn().mockReturnThis(),
toBuffer: vi.fn().mockResolvedValue(Buffer.from('fake-webp-buffer')),
toFile: vi.fn().mockImplementation(async (filePath: string) => {
fs.writeFileSync(filePath, 'fake-webp-data')
return {}
}),
metadata: vi.fn().mockResolvedValue({ width: 800, height: 600, format: 'png' }),
}
const sharpFunc = vi.fn(() => mockSharp)
@@ -265,7 +279,7 @@ describe('TranscodeService', () => {
})) as WorkflowTask
expect(task.assetId).toBe('asset-123')
expect(task.type).toBe('transcode')
expect(task.type).toBe('transcode_video')
expect(task.status).toBe('pending')
expect(task.payload?.projectId).toBe('proj-123')
expect(task.payload?.transcode?.videoStrategy).toBe('best_match')
@@ -347,4 +361,268 @@ describe('TranscodeService', () => {
expect.any(Function),
)
})
it('should generate PDF sprite sheet and poster using pdftoppm, ffmpeg, and sharp', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(
cmd: string,
args: string[],
cb: (err: Error | null, res: { stdout: string; stderr: string }) => void,
) => {
if (cmd === 'pdftoppm') {
// Create dummy page-1.png in temp directory (target output path prefix is args[args.length - 1])
const pagePrefix = args[args.length - 1]
const pagePath = `${pagePrefix}-1.png`
fs.writeFileSync(pagePath, 'fake-png-data')
cb(null, { stdout: '', stderr: '' })
} else if (cmd === 'pdfinfo') {
cb(null, { stdout: 'Title: Document\nPages: 15\nPage size: 612 x 792 pts', stderr: '' })
} else if (cmd === 'ffmpeg') {
cb(null, { stdout: '', stderr: '' })
} else {
cb(null, { stdout: '', stderr: '' })
}
},
)
const outputSprite = path.join(tempDir, 'sprite.webp')
const outputPoster = path.join(tempDir, 'poster.webp')
const result = await transcodeService.generatePdfSprite('input.pdf', outputSprite, outputPoster)
expect(result.pageCount).toBe(15)
expect(result.originalWidth).toBe(800)
expect(result.originalHeight).toBe(600)
expect(child_process.execFile).toHaveBeenCalledWith(
'pdftoppm',
expect.arrayContaining(['-png', '-f', '1', '-l', '100', 'input.pdf']),
expect.any(Function),
)
expect(child_process.execFile).toHaveBeenCalledWith(
'ffmpeg',
expect.arrayContaining(['-filter_complex', 'scale=w=480:h=-2,tile=10x10']),
expect.any(Function),
)
expect(child_process.execFile).toHaveBeenCalledWith(
'pdfinfo',
['input.pdf'],
expect.any(Function),
)
})
it('should extract PDF info metadata using pdftoppm and pdfinfo', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(
cmd: string,
args: string[],
cb: (err: Error | null, res: { stdout: string; stderr: string }) => void,
) => {
if (cmd === 'pdftoppm') {
const pagePrefix = args[args.length - 1]
const pagePath = `${pagePrefix}-1.png`
fs.writeFileSync(pagePath, 'fake-png-data')
cb(null, { stdout: '', stderr: '' })
} else if (cmd === 'pdfinfo') {
cb(null, { stdout: 'Pages: 42\n', stderr: '' })
} else {
cb(null, { stdout: '', stderr: '' })
}
},
)
const info = await transcodeService.getPdfInfo('document.pdf')
expect(info.mimeType).toBe('application/pdf')
expect(info.totalFrames).toBe(42)
expect(info.originalWidth).toBe(800)
expect(info.originalHeight).toBe(600)
expect(info.hasAudio).toBe(false)
})
it('should render PDF pages and upload to S3', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(
cmd: string,
args: string[],
cb: (err: Error | null, res: { stdout: string; stderr: string }) => void,
) => {
if (cmd === 'pdftoppm') {
const pagePrefix = args[args.length - 1]
const pagePath = `${pagePrefix}-1.png`
fs.writeFileSync(pagePath, 'fake-png-data')
cb(null, { stdout: '', stderr: '' })
} else {
cb(null, { stdout: '', stderr: '' })
}
},
)
vi.mocked(s3Service.downloadToFile).mockResolvedValue(undefined)
vi.mocked(s3Service.putObject).mockResolvedValue(
{} as unknown as Awaited<ReturnType<typeof s3Service.putObject>>,
)
const result = await transcodeService.renderPdfPages({
assetKey: 'projects/p1/doc.pdf',
assetId: 'asset-1',
start: 1,
end: 1,
})
expect(result.length).toBe(1)
expect(result[0].page).toBe(1)
expect(result[0].key).toContain('files/asset-1/pdf_pages/doc-page-1-')
expect(s3Service.putObject).toHaveBeenCalled()
})
describe('overlayAnnotationsOnBuffer Pixel Tests', () => {
it('should draw annotation overlay on a 100x200 image and mutate pixels', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actualSharp = (await vi.importActual('sharp')) as any
const realSharp = (actualSharp.default || actualSharp) as typeof sharp
const inputBuffer = await realSharp({
create: {
width: 100,
height: 200,
channels: 4,
background: { r: 255, g: 255, b: 255, alpha: 1 },
},
})
.png()
.toBuffer()
const annotations: PrismaJson.AnnotationList = [
{
type: 'box',
color: '#ff0000',
points: [
[0.2, 0.2],
[0.8, 0.8],
],
},
]
const sharpSpy = vi
.mocked(sharp)
.mockImplementation((input: unknown, options?: unknown) =>
realSharp(input as Parameters<typeof sharp>[0], options as Parameters<typeof sharp>[1]),
)
const outputBuffer = await transcodeService.overlayAnnotationsOnBuffer(
inputBuffer,
annotations,
)
sharpSpy.mockRestore()
const { data, info } = await realSharp(outputBuffer)
.raw()
.toBuffer({ resolveWithObject: true })
expect(info.width).toBe(100)
expect(info.height).toBe(200)
const pixelX = 50
const pixelY = 40
const offset = (pixelY * info.width + pixelX) * info.channels
const r = data[offset]
const g = data[offset + 1]
const b = data[offset + 2]
expect(r).toBeGreaterThan(150)
expect(g).toBeLessThan(120)
expect(b).toBeLessThan(120)
})
it('should draw annotation overlay on a 200x100 image and mutate pixels', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actualSharp = (await vi.importActual('sharp')) as any
const realSharp = (actualSharp.default || actualSharp) as typeof sharp
const inputBuffer = await realSharp({
create: {
width: 200,
height: 100,
channels: 4,
background: { r: 255, g: 255, b: 255, alpha: 1 },
},
})
.png()
.toBuffer()
const annotations: PrismaJson.AnnotationList = [
{
type: 'box',
color: '#00ff00',
points: [
[0.1, 0.1],
[0.9, 0.9],
],
},
]
const sharpSpy = vi
.mocked(sharp)
.mockImplementation((input: unknown, options?: unknown) =>
realSharp(input as Parameters<typeof sharp>[0], options as Parameters<typeof sharp>[1]),
)
const outputBuffer = await transcodeService.overlayAnnotationsOnBuffer(
inputBuffer,
annotations,
)
sharpSpy.mockRestore()
const { data, info } = await realSharp(outputBuffer)
.raw()
.toBuffer({ resolveWithObject: true })
expect(info.width).toBe(200)
expect(info.height).toBe(100)
const pixelX = 100
const pixelY = 10
const offset = (pixelY * info.width + pixelX) * info.channels
const r = data[offset]
const g = data[offset + 1]
const b = data[offset + 2]
expect(r).toBeLessThan(100)
expect(g).toBeGreaterThan(200)
expect(b).toBeLessThan(100)
})
it('should generate PDF from text file including CJK characters', async () => {
const txtFile = path.join(tempDir, 'test.txt')
const pdfFile = path.join(tempDir, 'output.pdf')
fs.writeFileSync(txtFile, 'Hello World\n你好世界\nこんにちは世界\n안녕하세요世界')
await transcodeService.generatePdfFromText(txtFile, pdfFile)
expect(fs.existsSync(pdfFile)).toBe(true)
const stat = fs.statSync(pdfFile)
expect(stat.size).toBeGreaterThan(0)
})
it('should generate PDF from CSV file including CJK characters', async () => {
const csvFile = path.join(tempDir, 'test.csv')
const pdfFile = path.join(tempDir, 'output.pdf')
fs.writeFileSync(
csvFile,
'Name,Age,Country\nHello World,25,USA\n你好世界,30,China\nこんにちは,28,Japan\n안녕하세요,22,Korea',
)
await transcodeService.generatePdfFromCsv(csvFile, pdfFile)
expect(fs.existsSync(pdfFile)).toBe(true)
const stat = fs.statSync(pdfFile)
expect(stat.size).toBeGreaterThan(0)
})
})
})
+476 -34
View File
@@ -1,10 +1,13 @@
import { s3Service } from '@shumai/core/src/s3/s3'
import { stemFromKey } from '@shumai/core/src/utils/filename'
import { prisma, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import '@shumai/db/src/prisma-json-types'
import { execFile } from 'child_process'
import { parse } from 'csv-parse/sync'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import PDFDocument from 'pdfkit'
import sharp from 'sharp'
import { ulid } from 'ulid'
import { promisify } from 'util'
@@ -12,6 +15,77 @@ import { dataFormatNames } from './dataFormatNames'
const execFileAsync = promisify(execFile)
export interface CjkFontConfig {
fontPath: string
fontName?: string
}
export function findCjkFontPath(): CjkFontConfig | undefined {
const candidates: { fontPath: string; fontName?: string }[] = [
// Custom env override
...(process.env.PDF_CJK_FONT_PATH
? [
{
fontPath: process.env.PDF_CJK_FONT_PATH,
fontName: process.env.PDF_CJK_FONT_NAME,
},
]
: []),
// macOS
{ fontPath: '/System/Library/Fonts/Supplemental/Arial Unicode.ttf' },
{ fontPath: '/Library/Fonts/Arial Unicode.ttf' },
{ fontPath: '/System/Library/Fonts/PingFang.ttc', fontName: 'PingFangSC-Regular' },
{ fontPath: '/System/Library/Fonts/STHeiti Light.ttc', fontName: 'STHeitiSC-Light' },
// Linux / Ubuntu / Debian Noto & WenQuanYi CJK
{
fontPath: '/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc',
fontName: 'NotoSansCJKsc-Regular',
},
{
fontPath: '/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc',
fontName: 'NotoSansCJKsc-Regular',
},
{ fontPath: '/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc', fontName: 'WenQuanYiZenHei' },
{ fontPath: '/usr/share/fonts/truetype/arphic/ukai.ttc', fontName: 'AR-PL-UKai-CN' },
{
fontPath: '/usr/share/fonts/noto/NotoSansCJK-Regular.ttc',
fontName: 'Noto Sans CJK SC',
},
// Lightweight Linux / Docker Droid Fallbacks
{ fontPath: '/usr/share/fonts/truetype/droid/DroidSansFallbackFull.ttf' },
{ fontPath: '/usr/share/fonts/truetype/droid/DroidSansFallback.ttf' },
{ fontPath: '/usr/share/fonts/google-droid/DroidSansFallback.ttf' },
// Windows CJK Fonts
{ fontPath: 'C:\\Windows\\Fonts\\msyh.ttc', fontName: 'MicrosoftYaHei' },
{ fontPath: 'C:\\Windows\\Fonts\\simsun.ttc', fontName: 'SimSun' },
{ fontPath: 'C:\\Windows\\Fonts\\simhei.ttf' },
]
for (const item of candidates) {
if (fs.existsSync(item.fontPath)) {
return item
}
}
return undefined
}
export function parseCsvContent(content: string): string[][] {
try {
/* eslint-disable @typescript-eslint/naming-convention */
return parse(content, {
skip_empty_lines: true,
relax_column_count: true,
relax_quotes: true,
trim: true,
delimiter_auto: true,
})
} catch {
return content
.split(/\r?\n/)
.filter((line) => line.trim().length > 0)
.map((line) => line.split(',').map((cell) => cell.trim()))
}
}
export interface MediaMetadata {
originalWidth: number
originalHeight: number
@@ -287,6 +361,271 @@ export class TranscodeService {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
async generatePdfSprite(
inputFile: string,
outputSprite: string,
outputPoster: string,
): Promise<{ pageCount: number; originalWidth: number; originalHeight: number }> {
const tmpDir = this.createTempDir('pdf-sprite-')
try {
const pagePrefix = path.join(tmpDir, 'page')
try {
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix])
} catch (err) {
const errCode = (err as Record<string, unknown>)?.code
const msg = err instanceof Error ? err.message : String(err)
const lower = msg.toLowerCase()
if (
errCode === 'ENOENT' ||
lower.includes('enoent') ||
(lower.includes('not found') && lower.includes('pdftoppm'))
) {
throw new Error(
`pdftoppm executable not found in $PATH. Please install poppler-utils / poppler. (${msg})`,
{ cause: err },
)
}
throw err
}
const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png'))
if (files.length === 0) {
throw new Error('pdftoppm produced no image outputs')
}
files.sort((a, b) => {
const numA = parseInt(a.replace(/[^0-9]/g, ''), 10)
const numB = parseInt(b.replace(/[^0-9]/g, ''), 10)
return numA - numB
})
const extractedCount = files.length
const firstPagePath = path.join(tmpDir, files[0])
const firstMeta = await sharp(firstPagePath, { limitInputPixels: false }).metadata()
const originalWidth = firstMeta.width || 800
const originalHeight = firstMeta.height || 1000
for (let f = 1; f <= 100; f++) {
const pageIdx = Math.min(extractedCount - 1, Math.floor(((f - 1) * extractedCount) / 100))
const srcPath = path.join(tmpDir, files[pageIdx])
const destPath = path.join(tmpDir, `frame_${f}.png`)
if (srcPath !== destPath) {
fs.copyFileSync(srcPath, destPath)
}
}
const spriteArgs = [
'-i',
path.join(tmpDir, 'frame_%d.png'),
'-filter_complex',
'scale=w=480:h=-2,tile=10x10',
'-frames:v',
'1',
'-c:v',
'libwebp',
'-q:v',
'75',
outputSprite,
]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs])
await sharp(firstPagePath, { limitInputPixels: false })
.resize(480, null, { withoutEnlargement: true, fit: 'inside' })
.webp({ quality: 75 })
.toFile(outputPoster)
let totalPages = extractedCount
try {
const { stdout } = await execFileAsync('pdfinfo', [inputFile])
const match = stdout.match(/Pages:\s+(\d+)/)
if (match) {
totalPages = parseInt(match[1], 10)
}
} catch {
// Fallback to extracted count if pdfinfo is unavailable
}
return {
pageCount: totalPages,
originalWidth,
originalHeight,
}
} finally {
this.removeDir(tmpDir)
}
}
async getPdfInfo(inputFile: string): Promise<MediaMetadata> {
const tmpDir = this.createTempDir('pdf-info-')
let originalWidth = 800
let originalHeight = 1000
let pageCount = 1
try {
const pagePrefix = path.join(tmpDir, 'page')
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '1', inputFile, pagePrefix])
const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png'))
if (files.length > 0) {
const meta = await sharp(path.join(tmpDir, files[0]), {
limitInputPixels: false,
}).metadata()
originalWidth = meta.width || 800
originalHeight = meta.height || 1000
}
} catch (err) {
console.warn('pdftoppm not available for PDF dimensions extraction:', err)
}
try {
const { stdout } = await execFileAsync('pdfinfo', [inputFile])
const match = stdout.match(/Pages:\s+(\d+)/)
if (match) {
pageCount = parseInt(match[1], 10)
}
} catch {
// Fallback
} finally {
this.removeDir(tmpDir)
}
return {
originalWidth,
originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: pageCount,
hasAudio: false,
mimeType: 'application/pdf',
}
}
async generatePdfFromText(inputFile: string, outputFile: string): Promise<void> {
const content = fs.readFileSync(inputFile, 'utf-8')
const containsCjk = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9fff\uac00-\ud7af]/.test(
content,
)
const cjkFont = containsCjk ? findCjkFontPath() : undefined
return new Promise<void>((resolve, reject) => {
const doc = new PDFDocument({
size: 'A4',
margin: 40,
bufferPages: true,
})
const writeStream = fs.createWriteStream(outputFile)
doc.pipe(writeStream)
if (cjkFont) {
if (cjkFont.fontName !== undefined) {
doc.font(cjkFont.fontPath, cjkFont.fontName)
} else {
doc.font(cjkFont.fontPath)
}
} else {
doc.font('Helvetica')
}
doc.fontSize(10)
doc.text(content, {
lineGap: 3,
paragraphGap: 4,
})
doc.end()
writeStream.on('finish', () => resolve())
writeStream.on('error', (err) => reject(err))
})
}
async generatePdfFromCsv(inputFile: string, outputFile: string): Promise<void> {
const content = fs.readFileSync(inputFile, 'utf-8')
const rows = parseCsvContent(content)
const containsCjk = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9fff\uac00-\ud7af]/.test(
content,
)
const cjkFont = containsCjk ? findCjkFontPath() : undefined
if (rows.length === 0) {
rows.push(['(Empty CSV)'])
}
const maxCols = Math.max(...rows.map((r) => r.length))
const isLandscape = maxCols > 5
return new Promise<void>((resolve, reject) => {
const doc = new PDFDocument({
size: 'A4',
layout: isLandscape ? 'landscape' : 'portrait',
margin: 30,
bufferPages: true,
})
const writeStream = fs.createWriteStream(outputFile)
doc.pipe(writeStream)
const setFont = () => {
if (cjkFont) {
if (cjkFont.fontName !== undefined) {
doc.font(cjkFont.fontPath, cjkFont.fontName)
} else {
doc.font(cjkFont.fontPath)
}
} else {
doc.font('Helvetica')
}
doc.fontSize(9)
}
setFont()
const pageWidth = doc.page.width - doc.page.margins.left - doc.page.margins.right
const colWidth = Math.max(40, pageWidth / maxCols)
const startX = doc.page.margins.left
let startY = doc.page.margins.top
const rowHeight = 20
const drawRow = (row: string[], isHeader: boolean, y: number) => {
if (isHeader) {
doc.rect(startX, y, colWidth * maxCols, rowHeight).fill('#e0e0e0')
doc.fillColor('#000000')
}
for (let col = 0; col < maxCols; col++) {
const text = row[col] || ''
const x = startX + col * colWidth
doc.rect(x, y, colWidth, rowHeight).stroke('#cccccc')
doc.fillColor('#000000').text(text, x + 4, y + 5, {
width: colWidth - 8,
height: rowHeight - 6,
ellipsis: true,
})
}
}
const headerRow = rows[0]
drawRow(headerRow, true, startY)
startY += rowHeight
for (let r = 1; r < rows.length; r++) {
if (startY + rowHeight > doc.page.height - doc.page.margins.bottom) {
doc.addPage()
setFont()
startY = doc.page.margins.top
drawRow(headerRow, true, startY)
startY += rowHeight
}
drawRow(rows[r], false, startY)
startY += rowHeight
}
doc.end()
writeStream.on('finish', () => resolve())
writeStream.on('error', (err) => reject(err))
})
}
async extractAudio(inputFile: string, outputFile: string, bitrate: string): Promise<void> {
const args = ['-i', inputFile, '-vn', '-acodec', 'libmp3lame', '-b:a', bitrate, outputFile]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
@@ -403,7 +742,7 @@ export class TranscodeService {
data: {
assetId,
projectId,
type: WorkflowTaskType.transcode,
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.pending,
payload: {
projectId,
@@ -418,7 +757,7 @@ export class TranscodeService {
data: {
assetId,
projectId,
type: WorkflowTaskType.transcode,
type: WorkflowTaskType.transcode_image,
status: WorkflowTaskStatus.pending,
payload: {
projectId,
@@ -503,18 +842,9 @@ export class TranscodeService {
params.annotations &&
params.annotations.length > 0
) {
const meta = await sharp(localShotPath, { limitInputPixels: false }).metadata()
const width = meta.width || 1280
const height = meta.height || 720
const svgStr = renderAnnotationsToSvg(width, height, params.annotations)
const tempCompositedPath = path.join(tmpDir, `composite-${outName}`)
await sharp(localShotPath, { limitInputPixels: false })
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
.toFile(tempCompositedPath)
fs.renameSync(tempCompositedPath, localShotPath)
const shotBuffer = fs.readFileSync(localShotPath)
const composited = await this.overlayAnnotationsOnBuffer(shotBuffer, params.annotations)
fs.writeFileSync(localShotPath, composited)
}
// 6. Upload to S3
@@ -531,41 +861,153 @@ export class TranscodeService {
}
}
async renderPdfPages(params: {
assetKey: string
assetId: string
start: number
end: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; page: number }>> {
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = this.createTempDir('pdf-pages-')
const pdfPath = path.join(tmpDir, path.basename(params.assetKey))
try {
// 1. Download PDF file
await s3Service.downloadToFile(bucket, params.assetKey, pdfPath)
// 2. Render pages via pdftoppm (-png -f start -l end)
const pagePrefix = path.join(tmpDir, 'page')
try {
await execFileAsync('pdftoppm', [
'-png',
'-f',
params.start.toString(),
'-l',
params.end.toString(),
pdfPath,
pagePrefix,
])
} catch (err) {
const errCode = (err as Record<string, unknown>)?.code
const msg = err instanceof Error ? err.message : String(err)
const lower = msg.toLowerCase()
if (
errCode === 'ENOENT' ||
lower.includes('enoent') ||
(lower.includes('not found') && lower.includes('pdftoppm'))
) {
throw new Error(
`pdftoppm executable not found in $PATH. Please install poppler-utils / poppler. (${msg})`,
{ cause: err },
)
}
throw err
}
const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png'))
if (files.length === 0) {
throw new Error('pdftoppm produced no image outputs')
}
files.sort((a, b) => {
const numA = parseInt(a.replace(/[^0-9]/g, ''), 10)
const numB = parseInt(b.replace(/[^0-9]/g, ''), 10)
return numA - numB
})
const results: Array<{ key: string; page: number }> = []
const stem = stemFromKey(params.assetKey)
for (let i = 0; i < files.length; i++) {
const pageNum = params.start + i
const localPngPath = path.join(tmpDir, files[i])
const webpName = `${stem}-page-${pageNum}-${ulid()}.webp`
let webpBuffer = await sharp(localPngPath, { limitInputPixels: false })
.resize(1920, 1080, { fit: 'inside', withoutEnlargement: false })
.webp({ quality: 85 })
.toBuffer()
// Overlay annotations if comment page matches pageNum
const commentPage =
params.commentTimestamp !== undefined && params.commentTimestamp !== null
? Math.round(params.commentTimestamp)
: null
if (
commentPage !== null &&
pageNum === commentPage &&
params.annotations &&
params.annotations.length > 0
) {
webpBuffer = await this.overlayAnnotationsOnBuffer(webpBuffer, params.annotations)
}
// Upload to S3 directly from buffer
const s3Key = `files/${params.assetId}/pdf_pages/${webpName}`
await s3Service.putObject(bucket, s3Key, webpBuffer, webpBuffer.length, 'image/webp')
results.push({ key: s3Key, page: pageNum })
}
return results
} finally {
this.removeDir(tmpDir)
}
}
async overlayAnnotationsOnBuffer(
imageBuffer: Buffer,
annotations: PrismaJson.AnnotationList,
): Promise<Buffer> {
if (!annotations || annotations.length === 0) {
return imageBuffer
}
const meta = await sharp(imageBuffer, { limitInputPixels: false }).metadata()
const width = meta.width || 1920
const height = meta.height || 1080
const svgStr = renderAnnotationsToSvg(width, height, annotations)
return await sharp(imageBuffer, { limitInputPixels: false })
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
.resize(16383, 16383, { fit: 'inside', withoutEnlargement: true })
.webp({ quality: 85 })
.toBuffer()
}
async overlayAnnotations(params: {
assetKey: string
assetId: string
annotations: PrismaJson.AnnotationList
}): Promise<string> {
const bucket = process.env.S3_BUCKET || 'shumai'
const outName = `annotation-${ulid()}.webp`
const tmpDir = this.createTempDir('annotation-')
const imgPath = path.join(tmpDir, path.basename(params.assetKey))
try {
// 1. Download image
await s3Service.downloadToFile(bucket, params.assetKey, imgPath)
const inputBuffer = fs.readFileSync(imgPath)
// 2. Read dimensions
const meta = await sharp(imgPath, { limitInputPixels: false }).metadata()
const width = meta.width || 1920
const height = meta.height || 1080
// 2. Overlay annotations in memory using helper method
const compositedBuffer = await this.overlayAnnotationsOnBuffer(
inputBuffer,
params.annotations,
)
// 3. Render SVG
const svgStr = renderAnnotationsToSvg(width, height, params.annotations)
// 4. Composite
const outName = `annotation-${ulid()}.webp`
const localOutPath = path.join(tmpDir, outName)
await sharp(imgPath, { limitInputPixels: false })
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
.resize(16383, 16383, { fit: 'inside', withoutEnlargement: true })
.webp({ quality: 90 })
.toFile(localOutPath)
// 5. Upload to S3
// 3. Upload to S3
const s3Key = `files/${params.assetId}/annotations/${outName}`
const fileBuffer = fs.readFileSync(localOutPath)
await s3Service.putObject(bucket, s3Key, fileBuffer, fileBuffer.length, 'image/webp')
await s3Service.putObject(
bucket,
s3Key,
compositedBuffer,
compositedBuffer.length,
'image/webp',
)
return s3Key
} finally {
+7 -7
View File
@@ -182,7 +182,7 @@ describe('UploadService', () => {
expect(updatedAsset?.status).toBe(AssetStatus.uploaded)
const workflowTask = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
})
expect(workflowTask).toBeDefined()
expect(workflowTask?.payload).toEqual({
@@ -219,7 +219,7 @@ describe('UploadService', () => {
await uploadService.confirmFileUpload(userId, task.id, { fileId: asset.id })
const workflowTask = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
where: { assetId: asset.id, type: WorkflowTaskType.transcode_image },
})
expect(workflowTask).toBeDefined()
expect(workflowTask?.payload).toEqual({
@@ -236,7 +236,7 @@ describe('UploadService', () => {
})
const asset = await prisma.asset.create({
data: {
name: 'file.txt',
name: 'file.json',
type: AssetType.file,
project: { connect: { id: projectId } },
parent: { connect: { id: parentId } },
@@ -247,7 +247,7 @@ describe('UploadService', () => {
create: { key: 'test-key' },
},
},
mediaType: 'text/plain',
mediaType: 'application/json',
},
})
@@ -257,7 +257,7 @@ describe('UploadService', () => {
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const workflowTask = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
})
expect(workflowTask).toBeNull()
})
@@ -290,7 +290,7 @@ describe('UploadService', () => {
expect(updatedAsset?.status).toBe(AssetStatus.uploaded)
const workflowTask = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
})
expect(workflowTask).toBeDefined()
})
@@ -323,7 +323,7 @@ describe('UploadService', () => {
expect(updatedAsset?.status).toBe(AssetStatus.uploaded)
const workflowTask = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
})
expect(workflowTask).toBeDefined()
})
+17 -5
View File
@@ -18,10 +18,11 @@ import {
PresignedUrl,
TaskInfo,
} from '@shumai/dtos'
import { ImageTranscoder, VideoTranscoder } from '@shumai/transcode'
import { ImageTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transcode'
import { generateKeyBetween } from 'jittered-fractional-indexing'
import { ulid } from 'ulid'
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
import { getProxyType } from '@shumai/core/src/utils/mime'
export class UploadService {
constructor(private readonly prismaClient: typeof prisma = prisma) {}
@@ -300,9 +301,14 @@ export class UploadService {
},
})
const isVideo = asset.mediaType?.startsWith('video/')
const isImage = asset.mediaType?.startsWith('image/')
const isAudio = asset.mediaType?.startsWith('audio/')
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
if (autofillAgent && (isVideo || isImage)) {
await tx.workflowTask.create({
@@ -320,7 +326,7 @@ export class UploadService {
})
}
if (!isVideo && !isImage && !isAudio) {
if (!proxyType) {
await tx.asset.update({
where: { id: asset.id },
data: { status: AssetStatus.processed },
@@ -341,6 +347,12 @@ export class UploadService {
} else if (isImage) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await new ImageTranscoder(tx as any, asset.id, team.id, projectId).withThumbnail().submit()
} else if (isPdf) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await new PdfTranscoder(tx as any, asset.id, team.id, projectId)
.withSprite()
.withPoster()
.submit()
}
}
}
+34 -1
View File
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest'
import { detectSupportedMimeType } from './mime'
import { detectSupportedMimeType, getProxyType } from './mime'
describe('detectSupportedMimeType', () => {
it('should detect JPEG', () => {
@@ -39,3 +39,36 @@ describe('detectSupportedMimeType', () => {
expect(detectSupportedMimeType(buffer)).toBeNull()
})
})
describe('getProxyType', () => {
it('should detect image proxyType', () => {
expect(getProxyType('image/png', 'test.png')).toBe('image')
expect(getProxyType('image/jpeg', 'photo.jpg')).toBe('image')
})
it('should detect video proxyType', () => {
expect(getProxyType('video/mp4', 'clip.mp4')).toBe('video')
})
it('should detect audio proxyType', () => {
expect(getProxyType('audio/mpeg', 'song.mp3')).toBe('audio')
})
it('should detect pdf proxyType only for pdf, csv, txt, and markdown files', () => {
expect(getProxyType('application/pdf', 'doc.pdf')).toBe('pdf')
expect(getProxyType('text/plain', 'notes.txt')).toBe('pdf')
expect(getProxyType('text/csv', 'data.csv')).toBe('pdf')
expect(getProxyType('text/markdown', 'README.md')).toBe('pdf')
expect(getProxyType('text/x-markdown', 'doc.markdown')).toBe('pdf')
expect(getProxyType(null, 'file.txt')).toBe('pdf')
expect(getProxyType(null, 'README.md')).toBe('pdf')
expect(getProxyType(null, 'doc.markdown')).toBe('pdf')
})
it('should return null for unsupported files including office files', () => {
expect(getProxyType('application/zip', 'archive.zip')).toBeNull()
expect(getProxyType('application/msword', 'letter.doc')).toBeNull()
expect(getProxyType(null, 'sheet.xlsx')).toBeNull()
expect(getProxyType(null, 'unknown.bin')).toBeNull()
})
})
+31
View File
@@ -1,3 +1,34 @@
export type ProxyType = 'image' | 'video' | 'audio' | 'pdf'
export function getProxyType(
mediaType?: string | null,
filename?: string | null,
): ProxyType | null {
const lowerMediaType = mediaType?.toLowerCase() || ''
const lowerFilename = filename?.toLowerCase() || ''
if (lowerMediaType.startsWith('image/')) return 'image'
if (lowerMediaType.startsWith('video/')) return 'video'
if (lowerMediaType.startsWith('audio/')) return 'audio'
if (
lowerMediaType === 'application/pdf' ||
lowerMediaType === 'text/plain' ||
lowerMediaType === 'text/csv' ||
lowerMediaType === 'text/markdown' ||
lowerMediaType === 'text/x-markdown' ||
lowerFilename.endsWith('.pdf') ||
lowerFilename.endsWith('.txt') ||
lowerFilename.endsWith('.csv') ||
lowerFilename.endsWith('.md') ||
lowerFilename.endsWith('.markdown')
) {
return 'pdf'
}
return null
}
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
export function detectSupportedMimeType(buffer: Uint8Array): string | null {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/db",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -0,0 +1,14 @@
-- AlterEnum
-- This migration adds more than one value to an enum.
-- With PostgreSQL versions 11 and earlier, this is not possible
-- in a single migration. This can be worked around by creating
-- multiple migrations, each migration adding only one value to
-- the enum.
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_video';
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_image';
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_pdf';
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_pdf_pages';
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_screenshot';
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_image_annotation';
@@ -0,0 +1,32 @@
-- Backfill media.proxyType for legacy assets
UPDATE "assets"
SET "media" = jsonb_set(
COALESCE("media"::jsonb, '{}'::jsonb),
'{proxyType}',
'"video"'::jsonb
)
WHERE "media" IS NOT NULL AND "media_type" LIKE 'video/%';
UPDATE "assets"
SET "media" = jsonb_set(
COALESCE("media"::jsonb, '{}'::jsonb),
'{proxyType}',
'"audio"'::jsonb
)
WHERE "media" IS NOT NULL AND "media_type" LIKE 'audio/%';
UPDATE "assets"
SET "media" = jsonb_set(
COALESCE("media"::jsonb, '{}'::jsonb),
'{proxyType}',
'"image"'::jsonb
)
WHERE "media" IS NOT NULL AND "media_type" LIKE 'image/%';
UPDATE "assets"
SET "media" = jsonb_set(
COALESCE("media"::jsonb, '{}'::jsonb),
'{proxyType}',
'"pdf"'::jsonb
)
WHERE "media" IS NOT NULL AND ("media_type" = 'application/pdf' OR "name" ILIKE '%.pdf');
+6
View File
@@ -100,6 +100,12 @@ enum TeamMemberScope {
enum WorkflowTaskType {
transcode
transcode_video
transcode_image
transcode_pdf
transcode_pdf_pages
transcode_screenshot
transcode_image_annotation
ai_metadata_autofill
chat
ai_embedding
+6
View File
@@ -126,6 +126,12 @@ export type TeamMemberScope = (typeof TeamMemberScope)[keyof typeof TeamMemberSc
export const WorkflowTaskType = {
transcode: 'transcode',
transcode_video: 'transcode_video',
transcode_image: 'transcode_image',
transcode_pdf: 'transcode_pdf',
transcode_pdf_pages: 'transcode_pdf_pages',
transcode_screenshot: 'transcode_screenshot',
transcode_image_annotation: 'transcode_image_annotation',
ai_metadata_autofill: 'ai_metadata_autofill',
chat: 'chat',
ai_embedding: 'ai_embedding',
File diff suppressed because one or more lines are too long
+16 -1
View File
@@ -124,19 +124,25 @@ declare global {
codec: string
}
export interface PdfTranscode {
key?: string
url?: string
}
export interface MediaInfo {
duration: number
filesize: number
frames: number
proxyType?: 'image' | 'video' | 'audio' | 'pdf'
imageTranscodes: ImageTranscode[]
videoTranscodes: VideoTranscode[]
videoPreview: VideoTranscode
pdfTranscode?: PdfTranscode
sprite?: SpriteInfo
poster?: PosterInfo
thumbnail?: ImageTranscode
finishedAt: string
metadata: Metadata | null
mimeType: string
original: OriginalInfo | null
}
@@ -255,6 +261,13 @@ declare global {
annotations?: AnnotationList | null
}
export interface PdfPagesSpec {
start: number
end: number
commentTimestamp?: number | null
annotations?: AnnotationList | null
}
export interface WorkflowTaskPayload {
projectId: string
transcode?: TaskSpec
@@ -265,6 +278,7 @@ declare global {
agentToolCall?: AgentToolCallPayload
screenshot?: ScreenshotSpec
imageAnnotation?: ImageAnnotationSpec
pdfPages?: PdfPagesSpec
}
// ----------------------------------------------------------------------
@@ -294,6 +308,7 @@ declare global {
attachedFiles?: string[]
assetIds?: string[]
isNewChat?: boolean
hasAssetChanged?: boolean
}
// ----------------------------------------------------------------------
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/dtos",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
+6 -4
View File
@@ -8,12 +8,13 @@ export const assetInfoPaginatedListSchema = z.object({
export type AssetInfoPaginatedList = z.infer<typeof assetInfoPaginatedListSchema>
export const previewInfoSchema = z.object({
mediaType: z.string().nullable(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
thumbnailUrl: z.string().optional(),
originalHeight: z.number().optional(),
originalWidth: z.number().optional(),
spriteUrl: z.string().optional(),
duration: z.number().optional(),
pageCount: z.number().optional(),
})
export type PreviewInfo = z.infer<typeof previewInfoSchema>
@@ -61,7 +62,7 @@ export const assetInfoSchema = z.object({
type: z.string(),
targetType: z.string().optional().nullable(),
status: z.string(),
mediaType: z.string().nullable(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
latestChildren: z.array(childPreviewSchema).optional(),
preview: previewInfoSchema.nullable().optional(),
createdAt: z.string(),
@@ -104,7 +105,8 @@ export const assetInfoSchema = z.object({
)
.optional(),
videoPreview: z.object({ url: z.string(), key: z.string().optional() }).optional(),
mimeType: z.string().optional(),
pdfTranscode: z.object({ url: z.string(), key: z.string().optional() }).optional(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
metadata: mediaMetadataSchema.optional(),
})
.optional(),
@@ -186,7 +188,7 @@ export const attachmentInfoSchema = z.object({
id: z.string(),
assetId: z.string(),
url: z.string(),
mediaType: z.string().nullable(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
})
export type AttachmentInfo = z.infer<typeof attachmentInfoSchema>
+1 -1
View File
@@ -18,7 +18,7 @@ export type EntityInfo = z.infer<typeof entityInfoSchema>
export const assetPreviewSchema = z.object({
id: z.string().optional(),
mediaType: z.string().optional(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
name: z.string().optional(),
preview: z.string().optional(),
thumbnailUrl: z.string().optional(),
Binary file not shown.
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/e2e",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"dependencies": {
@@ -230,6 +230,8 @@ describe.each(['local', 'temporal'] as const)(
type: 'file',
status: 'uploaded',
mediaType: 'image/png',
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
media: { proxyType: 'image' } as any,
projectId: project.id,
storageKeyId: storageKey.id,
},
+129
View File
@@ -325,4 +325,133 @@ describe.each(['local', 'temporal'] as const)('Workflow E2E - agentChat (executo
'direct-session-456',
)
}, 50000)
it('should run agentChat workflow with version stack asset successfully', async () => {
const team = await prisma.team.create({
data: { name: 'E2E Version Stack Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Version Stack Project', teamId: team.id },
})
const agentUser = await prisma.user.create({
data: {
name: 'E2E VS Agent User',
email: 'e2e-vs-agent@shumai.ai',
type: 'agent',
},
})
await prisma.teamMember.create({
data: {
teamId: team.id,
userId: agentUser.id,
role: 'editor',
},
})
const provider = await prisma.provider.create({
data: {
name: 'google',
teamId: team.id,
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
},
})
const model = await prisma.model.create({
data: {
modelId: 'gemini',
name: 'Gemini',
providerId: provider.id,
config: {
input: ['text', 'image'],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 8192,
maxTokens: 2048,
reasoning: false,
},
},
})
await prisma.agent.create({
data: {
id: agentUser.id,
teamId: team.id,
type: 'chat',
enabled: true,
providerId: provider.id,
modelId: model.id,
config: {
provider: 'google',
model: 'gemini',
},
},
})
const versionStack = await prisma.asset.create({
data: {
name: 'VersionStack Container',
type: 'version_stack',
status: 'uploaded',
projectId: project.id,
},
})
await prisma.asset.create({
data: {
name: 'version-1.png',
type: 'file',
status: 'uploaded',
mediaType: 'image/png',
projectId: project.id,
parentId: versionStack.id,
sortIndex: 'a0',
},
})
const regularUser = await prisma.user.create({
data: {
name: 'Regular User 3',
email: 'user3@example.com',
type: 'human',
},
})
await prisma.agentSession.create({
data: {
id: 'vs-session-789',
agentId: agentUser.id,
userId: regularUser.id,
cwd: process.cwd(),
assetId: versionStack.id,
},
})
const task = await prisma.workflowTask.create({
data: {
type: 'chat',
status: 'pending',
assetId: versionStack.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
agent: {
agentId: agentUser.id,
sessionId: 'vs-session-789',
prompt: 'Explain this version stack file',
userId: regularUser.id,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
expect((completedTask.output as unknown as { sessionId: string })?.sessionId).toBe(
'vs-session-789',
)
}, 50000)
})
+342 -11
View File
@@ -94,7 +94,7 @@ describe.each(['local', 'temporal'] as const)(
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode',
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
@@ -128,14 +128,14 @@ describe.each(['local', 'temporal'] as const)(
// Verify media info contains videoTranscodes, poster, sprite, and duration details
const mediaInfo = updatedAsset?.media as unknown as {
mimeType: string
proxyType: string
duration: number
videoTranscodes: { key: string }[]
poster: unknown
sprite: unknown
}
expect(mediaInfo).toBeDefined()
expect(mediaInfo.mimeType).toBe('video/mp4')
expect(mediaInfo.proxyType).toBe('video')
expect(mediaInfo.duration).toBeCloseTo(1.0, 1)
expect(mediaInfo.videoTranscodes).toBeDefined()
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
@@ -184,7 +184,7 @@ describe.each(['local', 'temporal'] as const)(
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode',
type: 'transcode_image',
status: 'pending',
assetId: asset.id,
projectId: project.id,
@@ -213,12 +213,12 @@ describe.each(['local', 'temporal'] as const)(
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
mimeType: string
proxyType: string
imageTranscodes: unknown[]
thumbnail: unknown
}
expect(mediaInfo).toBeDefined()
expect(mediaInfo.mimeType).toBe('image/png')
expect(mediaInfo.proxyType).toBe('image')
expect(mediaInfo.imageTranscodes).toBeDefined()
expect(mediaInfo.imageTranscodes.length).toBeGreaterThan(0)
expect(mediaInfo.thumbnail).toBeDefined()
@@ -265,7 +265,7 @@ describe.each(['local', 'temporal'] as const)(
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode',
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
@@ -292,11 +292,11 @@ describe.each(['local', 'temporal'] as const)(
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
mimeType: string
proxyType: string
videoTranscodes: { key: string }[]
}
expect(mediaInfo).toBeDefined()
expect(mediaInfo.mimeType).toBe('audio/wav')
expect(mediaInfo.proxyType).toBe('audio')
expect(mediaInfo.videoTranscodes).toBeDefined()
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
// Audio proxy key should end with -audio-proxy.mp4
@@ -344,7 +344,7 @@ describe.each(['local', 'temporal'] as const)(
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode',
type: 'transcode_screenshot',
status: 'pending',
assetId: asset.id,
projectId: project.id,
@@ -418,7 +418,7 @@ describe.each(['local', 'temporal'] as const)(
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode',
type: 'transcode_image_annotation',
status: 'pending',
assetId: asset.id,
projectId: project.id,
@@ -455,5 +455,336 @@ describe.each(['local', 'temporal'] as const)(
expect(output).toBeDefined()
expect(output.key).toContain('annotations/')
}, 50000)
it('should run transcodeMedia workflow for a PDF asset successfully', async () => {
// 1. Seed Database
const team = await prisma.team.create({
data: { name: 'E2E PDF Transcode Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E PDF Transcode Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: {
key: 'projects/e2e/test.pdf',
},
})
const asset = await prisma.asset.create({
data: {
name: 'test.pdf',
type: 'file',
status: 'uploaded',
mediaType: 'application/pdf',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
// 2. Seed S3 Storage from fixture test.pdf
const pdfPath = path.join(fixturesDir, 'test.pdf')
const pdfBuffer = fs.readFileSync(pdfPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/test.pdf',
pdfBuffer,
pdfBuffer.length,
'application/pdf',
)
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_pdf',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
poster: true,
sprite: true,
},
},
},
})
// 4. Wait for workflow to complete
console.log(
`Submitted E2E PDF Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
)
const completedTask = await workflowService.executeWait(task, 45000)
// 5. Verification
expect(completedTask.status).toBe('completed')
// Verify Asset status is updated to processed
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
// Verify media info contains pdf details, poster, sprite, and pageCount
const mediaInfo = updatedAsset?.media as unknown as {
proxyType: string
poster: { key: string }
sprite: { key: string }
frames?: number
pageCount?: number
metadata?: { totalFrames?: number; pageCount?: number }
}
expect(mediaInfo).toBeDefined()
expect(mediaInfo.proxyType).toBe('pdf')
expect(mediaInfo.poster).toBeDefined()
expect(mediaInfo.poster.key).toContain('poster.webp')
expect(mediaInfo.sprite).toBeDefined()
expect(mediaInfo.sprite.key).toContain('sprite.webp')
expect(mediaInfo.frames ?? mediaInfo.metadata?.totalFrames).toBeGreaterThan(0)
}, 50000)
it('should run renderPdfPages workflow for a PDF asset successfully', async () => {
// 1. Seed Database
const team = await prisma.team.create({
data: { name: 'E2E PDF Pages Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E PDF Pages Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: {
key: 'projects/e2e/test-pages.pdf',
},
})
const asset = await prisma.asset.create({
data: {
name: 'test-pages.pdf',
type: 'file',
status: 'uploaded',
mediaType: 'application/pdf',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
// 2. Seed S3 Storage from fixture test.pdf
const pdfPath = path.join(fixturesDir, 'test.pdf')
const pdfBuffer = fs.readFileSync(pdfPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/test-pages.pdf',
pdfBuffer,
pdfBuffer.length,
'application/pdf',
)
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_pdf_pages',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
pdfPages: {
start: 1,
end: 2,
},
},
},
})
// 4. Wait for workflow to complete
console.log(`Submitted E2E PDF Pages Workflow Task. ID: ${task.id}. Awaiting completion...`)
const completedTask = await workflowService.executeWait(task, 45000)
// 5. Verification
expect(completedTask.status).toBe('completed')
// Verify output contains rendered pages
const output = completedTask.output as unknown as { pages: { key: string; page: number }[] }
expect(output).toBeDefined()
expect(output.pages).toBeDefined()
expect(output.pages.length).toBeGreaterThan(0)
}, 50000)
it('should run transcodeMedia workflow for a TXT asset with CJK text to generate PDF proxy successfully', async () => {
// 1. Seed Database
const team = await prisma.team.create({
data: { name: 'E2E CJK TXT Transcode Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E CJK TXT Transcode Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: {
key: 'projects/e2e/cjk-test.txt',
},
})
const asset = await prisma.asset.create({
data: {
name: 'cjk-test.txt',
type: 'file',
status: 'uploaded',
mediaType: 'text/plain',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
// 2. Seed S3 Storage with CJK text
const cjkText =
'Hello World\n你好世界 (Chinese)\nこんにちは世界 (Japanese)\n안녕하세요世界 (Korean)\nLine 5 of random text.'
const txtBuffer = Buffer.from(cjkText, 'utf-8')
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/cjk-test.txt',
txtBuffer,
txtBuffer.length,
'text/plain',
)
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_pdf',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
poster: true,
sprite: true,
},
},
},
})
// 4. Wait for workflow to complete
console.log(
`Submitted E2E CJK TXT Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
)
const completedTask = await workflowService.executeWait(task, 45000)
// 5. Verification
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType?: string
pdfTranscode?: { key: string }
poster?: { key: string }
sprite?: { key: string }
frames?: number
}
expect(mediaInfo).toBeDefined()
expect(mediaInfo.proxyType).toBe('pdf')
expect(mediaInfo.pdfTranscode?.key).toBe(`files/${asset.id}/proxy.pdf`)
expect(mediaInfo.poster?.key).toContain('poster.webp')
expect(mediaInfo.sprite?.key).toContain('sprite.webp')
expect(mediaInfo.frames).toBeGreaterThan(0)
}, 50000)
it('should process markdown (.md) transcode PDF task correctly', async () => {
// 1. Create Team, Project, Asset & StorageKey
const team = await prisma.team.create({
data: { name: 'E2E MD Transcode Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E MD Transcode Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: {
key: 'projects/e2e/doc.md',
status: 'active',
},
})
const asset = await prisma.asset.create({
data: {
name: 'doc.md',
type: 'file',
status: 'uploaded',
mediaType: 'text/markdown',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
// 2. Seed S3 Storage with Markdown text
const mdText = '# Heading\n\nThis is a **markdown** document to be rendered to PDF proxy.'
const mdBuffer = Buffer.from(mdText, 'utf-8')
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/doc.md',
mdBuffer,
mdBuffer.length,
'text/markdown',
)
// 3. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_pdf',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
poster: true,
sprite: true,
},
},
},
})
// 4. Wait for workflow to complete
console.log(
`Submitted E2E MD Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
)
const completedTask = await workflowService.executeWait(task, 45000)
// 5. Verification
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType?: string
pdfTranscode?: { key: string }
poster?: { key: string }
sprite?: { key: string }
frames?: number
}
expect(mediaInfo).toBeDefined()
expect(mediaInfo.proxyType).toBe('pdf')
expect(mediaInfo.pdfTranscode?.key).toBe(`files/${asset.id}/proxy.pdf`)
expect(mediaInfo.poster?.key).toContain('poster.webp')
expect(mediaInfo.sprite?.key).toContain('sprite.webp')
expect(mediaInfo.frames).toBeGreaterThan(0)
}, 50000)
},
)
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/tableprinter",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/transcode",
"version": "0.1.10",
"version": "0.2.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -3,7 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { transcodeService } from '@shumai/core'
import { transcodeService } from '@shumai/core/src/transcode/transcode'
import * as child_process from 'child_process'
vi.mock('child_process', () => ({
@@ -12,6 +12,7 @@ vi.mock('child_process', () => ({
import {
getMediaInfoActivity,
overlayAnnotationsActivity,
renderPdfPagesActivity,
takeScreenshotsActivity,
transcodeVideoActivity,
transcodeAudioActivity,
@@ -19,6 +20,7 @@ import {
downloadMediaToTmpActivity,
transcodeImageActivity,
generateSpriteActivity,
generatePdfProxyActivity,
transcodeVideoChunkActivity,
deleteS3ObjectActivity,
} from './transcode'
@@ -39,14 +41,32 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({
getVideoInfo: vi.fn(),
getAudioInfo: vi.fn(),
getImageInfo: vi.fn(),
getPdfInfo: vi.fn().mockResolvedValue({
originalWidth: 800,
originalHeight: 1000,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: 10,
hasAudio: false,
mimeType: 'application/pdf',
}),
transcodeVideo: vi.fn(),
transcodeAudio: vi.fn(),
transcodeImage: vi.fn(),
generateSprite: vi.fn(),
generatePdfSprite: vi.fn().mockResolvedValue({
pageCount: 10,
originalWidth: 800,
originalHeight: 1000,
}),
generatePdfFromText: vi.fn(),
generatePdfFromCsv: vi.fn(),
createTempDir: vi.fn().mockReturnValue('/tmp'),
removeDir: vi.fn(),
takeScreenshots: vi.fn(),
overlayAnnotations: vi.fn(),
renderPdfPages: vi.fn(),
},
}))
@@ -97,6 +117,7 @@ describe('Transcode Activities', () => {
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/v.mp4',
proxyType: 'video',
mediaType: 'video/mp4',
})
@@ -133,6 +154,7 @@ describe('Transcode Activities', () => {
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/i.png',
proxyType: 'image',
mediaType: 'image/png',
})
@@ -168,6 +190,7 @@ describe('Transcode Activities', () => {
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/a.mp3',
proxyType: 'audio',
mediaType: 'audio/mpeg',
})
@@ -201,16 +224,73 @@ describe('Transcode Activities', () => {
await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/d.pdf',
proxyType: 'pdf',
mediaType: 'application/pdf',
})
expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith(
asset.id,
expect.arrayContaining([{ key: 'file_type', value: 'document' }]),
expect.arrayContaining([
{ key: 'file_type', value: 'document' },
{ key: 'proxy_type', value: 'pdf' },
]),
true,
)
})
it('should generate PDF proxy for text/csv files in generatePdfProxyActivity', async () => {
const asset = await prisma.asset.create({
data: { name: 'test.txt', type: 'file', status: 'uploaded' },
})
const generatePdfFromTextSpy = vi
.spyOn(transcodeService, 'generatePdfFromText')
.mockImplementation(async (_inPath, outPath) => {
const fs = await import('fs')
fs.writeFileSync(outPath, 'fake pdf content')
})
const res = await generatePdfProxyActivity({
assetId: asset.id,
assetKey: 'files/asset1/test.txt',
filePath: '/tmp/test.txt',
mediaType: 'text/plain',
filename: 'test.txt',
})
expect(res.pdfProxyKey).toBe(`files/${asset.id}/proxy.pdf`)
expect(generatePdfFromTextSpy).toHaveBeenCalled()
generatePdfFromTextSpy.mockRestore()
})
it('should generate PDF proxy for markdown files in generatePdfProxyActivity', async () => {
const asset = await prisma.asset.create({
data: { name: 'README.md', type: 'file', status: 'uploaded' },
})
const generatePdfFromTextSpy = vi
.spyOn(transcodeService, 'generatePdfFromText')
.mockImplementation(async (_inPath, outPath) => {
const fs = await import('fs')
fs.writeFileSync(outPath, 'fake pdf content from md')
})
const res = await generatePdfProxyActivity({
assetId: asset.id,
assetKey: 'files/asset1/README.md',
filePath: '/tmp/README.md',
mediaType: 'text/markdown',
filename: 'README.md',
})
expect(res.pdfProxyKey).toBe(`files/${asset.id}/proxy.pdf`)
expect(generatePdfFromTextSpy).toHaveBeenCalledWith(
'/tmp/README.md',
expect.stringContaining('proxy.pdf'),
)
generatePdfFromTextSpy.mockRestore()
})
it('should set file_type to file for unknown types', async () => {
const asset = await prisma.asset.create({
data: { name: 'u.xyz', type: 'file', status: 'uploaded' },
@@ -306,6 +386,31 @@ describe('Transcode Activities', () => {
expect(result).toBe('annotations/ann.webp')
})
it('should call transcodeService.renderPdfPages', async () => {
vi.mocked(transcodeService.renderPdfPages).mockResolvedValue([
{ key: 'pdf_pages/doc-page-1.webp', page: 1 },
])
const result = await renderPdfPagesActivity({
assetKey: 'doc.pdf',
assetId: 'asset-pdf',
start: 1,
end: 1,
commentTimestamp: 1,
annotations: [],
})
expect(transcodeService.renderPdfPages).toHaveBeenCalledWith({
assetKey: 'doc.pdf',
assetId: 'asset-pdf',
start: 1,
end: 1,
commentTimestamp: 1,
annotations: [],
})
expect(result).toEqual([{ key: 'pdf_pages/doc-page-1.webp', page: 1 }])
})
describe('Non-retryable Error Handling', () => {
it('should throw non-retryable ApplicationFailure when downloadMediaToTmpActivity fails with ENOENT', async () => {
vi.mocked(s3Service.downloadToFile).mockRejectedValue({
+140 -20
View File
@@ -1,8 +1,9 @@
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
import { transcodeService } from '@shumai/core/src/transcode/transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { stemFromKey } from '@shumai/core/src/utils/filename'
import { getProxyType } from '@shumai/core/src/utils/mime'
import { ApplicationFailure } from '@temporalio/activity'
import * as path from 'path'
import * as fs from 'fs'
@@ -24,7 +25,12 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
const record = err as Record<string, unknown>
return {
code: typeof record.code === 'string' ? record.code : undefined,
message: typeof record.message === 'string' ? record.message : String(err),
message:
typeof record.message === 'string'
? record.message
: err instanceof Error
? err.message
: String(err),
name: typeof record.name === 'string' ? record.name : undefined,
}
}
@@ -36,18 +42,16 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
export async function getMediaInfoActivity(params: {
filePath: string
assetId: string
mediaType: string
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
mediaType?: string
}): Promise<PrismaJson.MediaInfo> {
try {
const isVideo = params.mediaType.startsWith('video/')
const isImage = params.mediaType.startsWith('image/')
const isAudio = params.mediaType.startsWith('audio/')
const isDocument =
params.mediaType.startsWith('text/') ||
params.mediaType === 'application/pdf' ||
params.mediaType.includes('msword') ||
params.mediaType.includes('officedocument') ||
params.mediaType.includes('vnd.ms-')
const proxyType =
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
const fileType = isVideo
? 'video'
@@ -55,7 +59,7 @@ export async function getMediaInfoActivity(params: {
? 'audio'
: isImage
? 'image'
: isDocument
: isPdf
? 'document'
: 'file'
@@ -63,12 +67,12 @@ export async function getMediaInfoActivity(params: {
duration: 0,
filesize: 0,
frames: 0,
proxyType,
imageTranscodes: [],
videoTranscodes: [],
videoPreview: { width: 0, height: 0 },
finishedAt: new Date().toISOString(),
metadata: null,
mimeType: params.mediaType,
original: {
key: '', // Will be filled by caller or updated later
downloadUrl: '',
@@ -80,6 +84,9 @@ export async function getMediaInfoActivity(params: {
const metadataUpdates: { key: string; value: string | number }[] = [
{ key: 'file_type', value: fileType },
]
if (proxyType) {
metadataUpdates.push({ key: 'proxy_type', value: proxyType })
}
if (isVideo) {
const info = await transcodeService.getVideoInfo(params.filePath)
@@ -163,6 +170,25 @@ export async function getMediaInfoActivity(params: {
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
)
} else if (isPdf) {
const info = await transcodeService.getPdfInfo(params.filePath)
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: info.totalFrames,
startTimecode: '00:00:00:00',
hasAudio: false,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'total_pages', value: info.totalFrames },
)
}
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
@@ -401,12 +427,26 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
const posterFile = path.join(tmpDir, 'poster.webp')
try {
await transcodeService.generateSprite(
params.filePath,
spriteFile,
posterFile,
params.mediaInfo.duration,
)
if (params.mediaInfo.proxyType === 'pdf') {
const pdfRes = await transcodeService.generatePdfSprite(
params.filePath,
spriteFile,
posterFile,
)
if (params.mediaInfo && params.mediaInfo.metadata) {
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
params.mediaInfo.frames = pdfRes.pageCount
}
} else {
await transcodeService.generateSprite(
params.filePath,
spriteFile,
posterFile,
params.mediaInfo.duration,
)
}
const spriteBuffer = fs.readFileSync(spriteFile)
await s3Service.putObject(
@@ -452,6 +492,56 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
}
}
export interface GeneratePdfProxyActivityParams {
assetId: string
assetKey: string
filePath: string
mediaType: string
filename: string
}
export async function generatePdfProxyActivity(
params: GeneratePdfProxyActivityParams,
): Promise<{ pdfProxyKey: string; pdfFilePath: string }> {
const isTxt =
params.mediaType === 'text/plain' ||
params.mediaType === 'text/markdown' ||
params.mediaType === 'text/x-markdown' ||
params.filename.toLowerCase().endsWith('.txt') ||
params.filename.toLowerCase().endsWith('.md') ||
params.filename.toLowerCase().endsWith('.markdown')
const isCsv = params.mediaType === 'text/csv' || params.filename.toLowerCase().endsWith('.csv')
if (!isTxt && !isCsv) {
return { pdfProxyKey: params.assetKey, pdfFilePath: params.filePath }
}
const bucket = process.env.S3_BUCKET || 'shumai'
const pdfProxyKey = `files/${params.assetId}/proxy.pdf`
const tmpDir = path.dirname(params.filePath)
const pdfFilePath = path.join(tmpDir, 'proxy.pdf')
try {
if (isCsv) {
await transcodeService.generatePdfFromCsv(params.filePath, pdfFilePath)
} else {
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
}
const buffer = fs.readFileSync(pdfFilePath)
await s3Service.putObject(bucket, pdfProxyKey, buffer, buffer.length, 'application/pdf')
return { pdfProxyKey, pdfFilePath }
} catch (err) {
const { message } = getErrorDetails(err)
throw ApplicationFailure.create({
message: `Failed to generate PDF proxy for text/csv/markdown: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
}
export async function downloadMediaToTmpActivity(params: {
assetKey: string
}): Promise<{ filePath: string; tmpDir: string }> {
@@ -573,6 +663,36 @@ export async function overlayAnnotationsActivity(params: {
}
}
export async function renderPdfPagesActivity(params: {
assetKey: string
assetId: string
start: number
end: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; page: number }>> {
try {
return await transcodeService.renderPdfPages(params)
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('pdftoppm') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `PDF page rendering failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export interface CreateEmbeddingTaskIfEnabledParams {
assetId: string
teamId: string | null
+12
View File
@@ -1,10 +1,22 @@
import { WorkflowTaskType } from '@shumai/db'
import { registerActivities, registerWorkflow } from '@shumai/workflow-core'
import * as transcodeActivities from './activities/transcode'
import { transcodeVideoWorkflow } from './workflows/transcode-video'
import { transcodeImageWorkflow } from './workflows/transcode-image'
import { transcodePdfWorkflow } from './workflows/transcode-pdf'
import { renderPdfPagesWorkflow } from './workflows/render-pdf-pages'
import { takeVideoScreenshotsWorkflow } from './workflows/take-video-screenshots'
import { overlayImageAnnotationWorkflow } from './workflows/overlay-image-annotation'
import { transcodeMedia } from './workflows/transcode'
export function initTranscodeWorkflows() {
registerWorkflow(WorkflowTaskType.transcode, transcodeMedia)
registerWorkflow(WorkflowTaskType.transcode_video, transcodeVideoWorkflow)
registerWorkflow(WorkflowTaskType.transcode_image, transcodeImageWorkflow)
registerWorkflow(WorkflowTaskType.transcode_pdf, transcodePdfWorkflow)
registerWorkflow(WorkflowTaskType.transcode_pdf_pages, renderPdfPagesWorkflow)
registerWorkflow(WorkflowTaskType.transcode_screenshot, takeVideoScreenshotsWorkflow)
registerWorkflow(WorkflowTaskType.transcode_image_annotation, overlayImageAnnotationWorkflow)
registerActivities(transcodeActivities)
}
+40 -2
View File
@@ -38,7 +38,7 @@ export class VideoTranscoder {
assetId: this.assetId,
teamId: this.teamId,
projectId: this.projectId,
type: WorkflowTaskType.transcode,
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.pending,
payload: {
projectId: this.projectId,
@@ -71,7 +71,45 @@ export class ImageTranscoder {
assetId: this.assetId,
teamId: this.teamId,
projectId: this.projectId,
type: WorkflowTaskType.transcode,
type: WorkflowTaskType.transcode_image,
status: WorkflowTaskStatus.pending,
payload: {
projectId: this.projectId,
transcode: this.spec,
},
},
})
return task.id
}
}
export class PdfTranscoder {
private spec: PrismaJson.TaskSpec = {}
constructor(
private readonly db: TransactionClient,
private readonly assetId: string,
private readonly teamId: string,
private readonly projectId: string,
) {}
withSprite(): this {
this.spec.sprite = true
return this
}
withPoster(): this {
this.spec.poster = true
return this
}
async submit(): Promise<string> {
const task = await this.db.workflowTask.create({
data: {
assetId: this.assetId,
teamId: this.teamId,
projectId: this.projectId,
type: WorkflowTaskType.transcode_pdf,
status: WorkflowTaskStatus.pending,
payload: {
projectId: this.projectId,
@@ -0,0 +1,61 @@
import type { WorkflowTask } from '@shumai/db'
import { executeActivity, getActivities, TaskQueueTranscode } from '@shumai/workflow-core'
import { ApplicationFailure } from '@temporalio/workflow'
export async function getWorkerQueueAndStartTask(task: WorkflowTask): Promise<string> {
const { getTranscodeWorkerQueueActivity, updateTaskStatusActivity } = getActivities()
const workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity)
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId: task.id,
status: 'processing',
})
return workerQueue
}
export async function fetchAssetWithKey(workerQueue: string, assetId: string) {
const { getAssetActivity } = getActivities()
const asset = await executeActivity(workerQueue, getAssetActivity, assetId)
const key = asset?.storageKey?.key
if (!asset || !key) {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
nonRetryable: true,
})
}
return { asset, key }
}
export async function completeTask(
workerQueue: string,
taskId: string,
output?: Record<string, unknown>,
) {
const { updateTaskStatusActivity } = getActivities()
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId,
status: 'completed',
...(output ? { output } : {}),
})
}
export async function failTask(workerQueue: string, taskId: string, err: unknown) {
const { updateTaskStatusActivity } = getActivities()
if (workerQueue) {
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId,
status: 'failed',
output: { error: err instanceof Error ? err.message : String(err) },
})
}
}
export async function cleanupTmpDir(workerQueue: string, tmpDir: string | undefined) {
if (tmpDir && workerQueue) {
const { cleanupTmpDirActivity } = getActivities()
try {
await executeActivity(workerQueue, cleanupTmpDirActivity, { tmpDir })
} catch (cleanupErr) {
console.error('Failed to cleanup tmp dir:', cleanupErr)
}
}
}
@@ -0,0 +1,101 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { overlayImageAnnotationWorkflow } from './overlay-image-annotation'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
sleep: vi.fn(),
}
})
describe('overlayImageAnnotationWorkflow', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
overlayAnnotationsActivity: Object.assign(vi.fn(), {
_activityName: 'overlayAnnotationsActivity',
}),
}
beforeEach(() => {
vi.clearAllMocks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.executeActivity as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(_queue: string, fn: any, ...args: any[]) => {
if (typeof fn !== 'function') {
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
}
return fn(...args)
},
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
})
it('should run image annotation overlay workflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-image-ann',
assetId: 'asset-image',
type: WorkflowTaskType.transcode_image_annotation,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
imageAnnotation: {
annotations: [{ type: 'box', color: '#ff0000', points: [] }],
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-image',
storageKey: { key: 'image.png' },
mediaType: 'image/png',
media: {
imageTranscodes: [{ key: 'image-transcoded.webp' }],
},
})
mockActivities.overlayAnnotationsActivity.mockResolvedValue(
'files/asset-image/annotations/ann1.webp',
)
await overlayImageAnnotationWorkflow(task)
expect(mockActivities.overlayAnnotationsActivity).toHaveBeenCalledWith({
assetKey: 'image-transcoded.webp',
assetId: 'asset-image',
annotations: [{ type: 'box', color: '#ff0000', points: [] }],
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-image-ann',
status: WorkflowTaskStatus.completed,
output: { key: 'files/asset-image/annotations/ann1.webp' },
})
})
})
@@ -0,0 +1,49 @@
import type { WorkflowTask } from '@shumai/db'
import '@shumai/db/src/prisma-json-types'
import { executeActivity, getActivities } from '@shumai/workflow-core'
import { ApplicationFailure } from '@temporalio/workflow'
import { getWorkerQueueAndStartTask, completeTask, failTask } from './common'
export async function overlayImageAnnotationWorkflow(task: WorkflowTask): Promise<void> {
let workerQueue = ''
try {
workerQueue = await getWorkerQueueAndStartTask(task)
const payload = task.payload
if (!payload?.imageAnnotation) {
throw ApplicationFailure.create({
message: 'Invalid payload for imageAnnotation task',
nonRetryable: true,
})
}
const { getAssetActivity, overlayAnnotationsActivity } = getActivities()
const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId)
let key = asset?.storageKey?.key
if (asset?.media) {
const mediaInfo = asset.media as unknown as PrismaJson.MediaInfo
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
key = mediaInfo.imageTranscodes[0].key || key
}
}
if (!asset || !key) {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
nonRetryable: true,
})
}
const keyWithOverlay = await executeActivity(workerQueue, overlayAnnotationsActivity, {
assetKey: key,
assetId: asset.id,
annotations: payload.imageAnnotation.annotations,
})
await completeTask(workerQueue, task.id, { key: keyWithOverlay })
} catch (err) {
console.error(`overlayImageAnnotationWorkflow failed for task ${task.id}:`, err)
await failTask(workerQueue, task.id, err)
throw err
}
}
@@ -0,0 +1,110 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { renderPdfPagesWorkflow } from './render-pdf-pages'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
sleep: vi.fn(),
}
})
describe('renderPdfPagesWorkflow', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
renderPdfPagesActivity: Object.assign(vi.fn(), {
_activityName: 'renderPdfPagesActivity',
}),
}
beforeEach(() => {
vi.clearAllMocks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.executeActivity as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(_queue: string, fn: any, ...args: any[]) => {
if (typeof fn !== 'function') {
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
}
return fn(...args)
},
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
})
it('should run renderPdfPagesWorkflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-pdf-pages',
assetId: 'asset-pdf',
type: WorkflowTaskType.transcode_pdf_pages,
status: WorkflowTaskStatus.pending,
payload: {
projectId: 'proj-1',
pdfPages: {
start: 1,
end: 2,
commentTimestamp: 2,
annotations: [],
},
},
output: null,
sessionId: null,
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-pdf',
storageKey: { key: 'doc.pdf' },
mediaType: 'application/pdf',
})
mockActivities.renderPdfPagesActivity.mockResolvedValue([
{ key: 'pdf_pages/doc-page-1.webp', page: 1 },
{ key: 'pdf_pages/doc-page-2.webp', page: 2 },
])
await renderPdfPagesWorkflow(task)
expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalledWith({
assetKey: 'doc.pdf',
assetId: 'asset-pdf',
start: 1,
end: 2,
commentTimestamp: 2,
annotations: [],
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-pdf-pages',
status: WorkflowTaskStatus.completed,
output: {
pages: [
{ key: 'pdf_pages/doc-page-1.webp', page: 1 },
{ key: 'pdf_pages/doc-page-2.webp', page: 2 },
],
},
})
})
})
@@ -0,0 +1,38 @@
import type { WorkflowTask } from '@shumai/db'
import { executeActivity, getActivities } from '@shumai/workflow-core'
import { ApplicationFailure } from '@temporalio/workflow'
import { getWorkerQueueAndStartTask, fetchAssetWithKey, completeTask, failTask } from './common'
export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void> {
let workerQueue = ''
try {
workerQueue = await getWorkerQueueAndStartTask(task)
const payload = task.payload
if (!payload?.pdfPages) {
throw ApplicationFailure.create({
message: 'Invalid payload for pdfPages task',
nonRetryable: true,
})
}
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
const { renderPdfPagesActivity } = getActivities()
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
assetKey: pdfKey,
assetId: asset.id,
start: payload.pdfPages.start,
end: payload.pdfPages.end,
commentTimestamp: payload.pdfPages.commentTimestamp,
annotations: payload.pdfPages.annotations,
})
await completeTask(workerQueue, task.id, { pages })
} catch (err) {
console.error(`renderPdfPagesWorkflow failed for task ${task.id}:`, err)
await failTask(workerQueue, task.id, err)
throw err
}
}
@@ -0,0 +1,110 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { takeVideoScreenshotsWorkflow } from './take-video-screenshots'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
sleep: vi.fn(),
}
})
describe('takeVideoScreenshotsWorkflow', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
takeScreenshotsActivity: Object.assign(vi.fn(), { _activityName: 'takeScreenshotsActivity' }),
}
beforeEach(() => {
vi.clearAllMocks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.executeActivity as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(_queue: string, fn: any, ...args: any[]) => {
if (typeof fn !== 'function') {
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
}
return fn(...args)
},
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
})
it('should run screenshot workflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-screenshot',
assetId: 'asset-video',
type: WorkflowTaskType.transcode_screenshot,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
screenshot: {
start: 0,
end: 10,
count: 2,
commentTimestamp: 5.0,
annotations: [],
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-video',
storageKey: { key: 'video.mp4' },
mediaType: 'video/mp4',
})
mockActivities.takeScreenshotsActivity.mockResolvedValue([
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
])
await takeVideoScreenshotsWorkflow(task)
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
assetKey: 'video.mp4',
assetId: 'asset-video',
start: 0,
end: 10,
count: 2,
commentTimestamp: 5.0,
annotations: [],
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-screenshot',
status: WorkflowTaskStatus.completed,
output: {
screenshots: [
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
],
},
})
})
})
@@ -0,0 +1,38 @@
import type { WorkflowTask } from '@shumai/db'
import { executeActivity, getActivities } from '@shumai/workflow-core'
import { ApplicationFailure } from '@temporalio/workflow'
import { getWorkerQueueAndStartTask, fetchAssetWithKey, completeTask, failTask } from './common'
export async function takeVideoScreenshotsWorkflow(task: WorkflowTask): Promise<void> {
let workerQueue = ''
try {
workerQueue = await getWorkerQueueAndStartTask(task)
const payload = task.payload
if (!payload?.screenshot) {
throw ApplicationFailure.create({
message: 'Invalid payload for screenshot task',
nonRetryable: true,
})
}
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
const { takeScreenshotsActivity } = getActivities()
const screenshots = await executeActivity(workerQueue, takeScreenshotsActivity, {
assetKey: key,
assetId: asset.id,
start: payload.screenshot.start,
end: payload.screenshot.end,
count: payload.screenshot.count,
commentTimestamp: payload.screenshot.commentTimestamp,
annotations: payload.screenshot.annotations,
})
await completeTask(workerQueue, task.id, { screenshots })
} catch (err) {
console.error(`takeVideoScreenshotsWorkflow failed for task ${task.id}:`, err)
await failTask(workerQueue, task.id, err)
throw err
}
}
@@ -0,0 +1,149 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { transcodeImageWorkflow } from './transcode-image'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
sleep: vi.fn(),
}
})
describe('transcodeImageWorkflow', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
}),
updateAssetStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getMediaInfoActivity: Object.assign(vi.fn(), { _activityName: 'getMediaInfoActivity' }),
transcodeImageActivity: Object.assign(vi.fn(), { _activityName: 'transcodeImageActivity' }),
updateAssetMediaActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetMediaActivity',
}),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
downloadMediaToTmpActivity: Object.assign(vi.fn(), {
_activityName: 'downloadMediaToTmpActivity',
}),
cleanupTmpDirActivity: Object.assign(vi.fn(), { _activityName: 'cleanupTmpDirActivity' }),
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
}),
}
beforeEach(() => {
vi.clearAllMocks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.executeActivity as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(_queue: string, fn: any, ...args: any[]) => {
if (typeof fn !== 'function') {
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
}
return fn(...args)
},
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/image.jpg',
tmpDir: '/tmp',
})
})
it('should process image transcode and thumbnail successfully', async () => {
const task: WorkflowTask = {
id: 'task-image',
assetId: 'asset-image',
type: WorkflowTaskType.transcode_image,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {
thumbnail: true,
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-image',
storageKey: { key: 'image.jpg' },
mediaType: 'image/jpeg',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
proxyType: 'image',
metadata: {
originalWidth: 1000,
originalHeight: 1000,
duration: 0,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
bitRate: 0,
hasAudio: false,
format: {},
},
videoTranscodes: [],
imageTranscodes: [],
})
mockActivities.transcodeImageActivity.mockResolvedValue({
key: 't.webp',
width: 480,
height: 480,
format: 'webp',
})
await transcodeImageWorkflow(task)
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
status: AssetStatus.processing,
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
mediaInfo: expect.objectContaining({
thumbnail: expect.objectContaining({
key: 't.webp',
width: 480,
height: 480,
}),
}),
})
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
status: AssetStatus.processed,
})
expect(mockActivities.createEmbeddingTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
teamId: 'team-1',
projectId: 'proj-1',
})
})
})
@@ -0,0 +1,107 @@
import type { WorkflowTask } from '@shumai/db'
import '@shumai/db/src/prisma-json-types'
import { executeActivity, getActivities } from '@shumai/workflow-core'
import {
getWorkerQueueAndStartTask,
fetchAssetWithKey,
completeTask,
failTask,
cleanupTmpDir,
} from './common'
export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void> {
let tmpDir: string | undefined
let workerQueue = ''
try {
workerQueue = await getWorkerQueueAndStartTask(task)
const {
updateAssetStatusActivity,
getMediaInfoActivity,
transcodeImageActivity,
updateAssetMediaActivity,
downloadMediaToTmpActivity,
createEmbeddingTaskIfEnabledActivity,
} = getActivities()
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: task.assetId,
status: 'processing',
})
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, {
assetKey: key,
})
const { filePath } = download
tmpDir = download.tmpDir
const spec = task.payload?.transcode || {}
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath,
assetId: asset.id,
proxyType: 'image',
mediaType: asset.mediaType || '',
})
mediaInfo.original = {
key,
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
}
const metadata = mediaInfo.metadata
const isImage = mediaInfo.proxyType === 'image'
if (isImage && metadata) {
const imageSpec: PrismaJson.ImageTranscode = {
width: metadata.originalWidth,
height: metadata.originalHeight,
quality: 90,
format: 'webp',
}
const imageTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: key,
filePath,
imageSpec,
})
mediaInfo.imageTranscodes.push(imageTranscode)
}
if (spec.thumbnail) {
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: key,
filePath,
imageSpec: { width: 480, height: 0, quality: 80, format: 'webp' },
})
mediaInfo.thumbnail = thumbTranscode
}
await executeActivity(workerQueue, updateAssetMediaActivity, {
assetId: asset.id,
mediaInfo,
})
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: asset.id,
status: 'processed',
})
await executeActivity(workerQueue, createEmbeddingTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
await completeTask(workerQueue, task.id)
} catch (err) {
console.error(`transcodeImageWorkflow failed for task ${task.id}:`, err)
await failTask(workerQueue, task.id, err)
throw err
} finally {
await cleanupTmpDir(workerQueue, tmpDir)
}
}
@@ -0,0 +1,154 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { transcodePdfWorkflow } from './transcode-pdf'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
sleep: vi.fn(),
}
})
describe('transcodePdfWorkflow', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
}),
updateAssetStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getMediaInfoActivity: Object.assign(vi.fn(), { _activityName: 'getMediaInfoActivity' }),
generateSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generateSpriteActivity' }),
updateAssetMediaActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetMediaActivity',
}),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
downloadMediaToTmpActivity: Object.assign(vi.fn(), {
_activityName: 'downloadMediaToTmpActivity',
}),
generatePdfProxyActivity: Object.assign(vi.fn(), {
_activityName: 'generatePdfProxyActivity',
}),
cleanupTmpDirActivity: Object.assign(vi.fn(), { _activityName: 'cleanupTmpDirActivity' }),
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
}),
}
beforeEach(() => {
vi.clearAllMocks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.executeActivity as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(_queue: string, fn: any, ...args: any[]) => {
if (typeof fn !== 'function') {
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
}
return fn(...args)
},
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/doc.pdf',
tmpDir: '/tmp',
})
mockActivities.generatePdfProxyActivity.mockResolvedValue({
pdfProxyKey: 'document.pdf',
pdfFilePath: '/tmp/doc.pdf',
})
})
it('should process pdf transcode and sprite generation successfully', async () => {
const task: WorkflowTask = {
id: 'task-pdf',
assetId: 'asset-pdf',
type: WorkflowTaskType.transcode_pdf,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {
sprite: true,
poster: true,
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid-pdf',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-pdf',
storageKey: { key: 'document.pdf' },
mediaType: 'application/pdf',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
proxyType: 'pdf',
metadata: {
originalWidth: 800,
originalHeight: 1000,
duration: 0,
frameRate: 0,
totalFrames: 12,
startTimecode: '00:00:00:00',
bitRate: 0,
hasAudio: false,
format: {},
},
videoTranscodes: [],
imageTranscodes: [],
})
mockActivities.generateSpriteActivity.mockResolvedValue({
sprite: { key: 'sprite.webp', frames: 100, tileX: 10, tileY: 10 },
poster: { key: 'poster.webp' },
})
await transcodePdfWorkflow(task)
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
status: AssetStatus.processing,
})
expect(mockActivities.generateSpriteActivity).toHaveBeenCalledWith({
assetKey: 'document.pdf',
filePath: '/tmp/doc.pdf',
spriteSpec: expect.objectContaining({ key: 'sprite.webp' }),
posterSpec: expect.objectContaining({ key: 'poster.webp' }),
mediaInfo: expect.objectContaining({ proxyType: 'pdf' }),
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
mediaInfo: expect.objectContaining({
sprite: { key: 'sprite.webp', frames: 100, tileX: 10, tileY: 10 },
poster: { key: 'poster.webp' },
}),
})
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
status: AssetStatus.processed,
})
})
})
@@ -0,0 +1,119 @@
import type { WorkflowTask } from '@shumai/db'
import '@shumai/db/src/prisma-json-types'
import { executeActivity, getActivities } from '@shumai/workflow-core'
import {
getWorkerQueueAndStartTask,
fetchAssetWithKey,
completeTask,
failTask,
cleanupTmpDir,
} from './common'
export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
let tmpDir: string | undefined
let workerQueue = ''
try {
workerQueue = await getWorkerQueueAndStartTask(task)
const {
updateAssetStatusActivity,
getMediaInfoActivity,
generateSpriteActivity,
updateAssetMediaActivity,
downloadMediaToTmpActivity,
generatePdfProxyActivity,
createEmbeddingTaskIfEnabledActivity,
} = getActivities()
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: task.assetId,
status: 'processing',
})
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, {
assetKey: key,
})
let currentFilePath = download.filePath
tmpDir = download.tmpDir
const pdfProxy = await executeActivity(workerQueue, generatePdfProxyActivity, {
assetId: asset.id,
assetKey: key,
filePath: currentFilePath,
mediaType: asset.mediaType || '',
filename: asset.name || '',
})
currentFilePath = pdfProxy.pdfFilePath
const spec = task.payload?.transcode || {}
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath: currentFilePath,
assetId: asset.id,
proxyType: 'pdf',
mediaType: asset.mediaType || '',
})
mediaInfo.original = {
key,
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
}
mediaInfo.pdfTranscode = {
key: pdfProxy.pdfProxyKey,
}
if (spec.sprite || spec.poster) {
const lastSlashIndex = key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
const spriteSpec: PrismaJson.SpriteInfo = {
key: assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp',
frames: 100,
tileX: 10,
tileY: 10,
}
const posterSpec: PrismaJson.PosterInfo = {
key: assetDir ? `${assetDir}/poster.webp` : 'poster.webp',
}
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
assetKey: key,
filePath: currentFilePath,
spriteSpec,
posterSpec,
mediaInfo,
})
mediaInfo.sprite = spriteResult.sprite
mediaInfo.poster = spriteResult.poster
}
await executeActivity(workerQueue, updateAssetMediaActivity, {
assetId: asset.id,
mediaInfo,
})
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: asset.id,
status: 'processed',
})
await executeActivity(workerQueue, createEmbeddingTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
await completeTask(workerQueue, task.id)
} catch (err) {
console.error(`transcodePdfWorkflow failed for task ${task.id}:`, err)
await failTask(workerQueue, task.id, err)
throw err
} finally {
await cleanupTmpDir(workerQueue, tmpDir)
}
}
@@ -0,0 +1,98 @@
import '@shumai/db/src/prisma-json-types'
export function isMimePsd(mimeType: string): boolean {
switch (mimeType) {
case 'image/vnd.adobe.photoshop':
case 'image/x-photoshop':
case 'application/x-photoshop':
case 'image/psd':
return true
}
return false
}
const RESOLUTION_LONG_SIDES: Record<string, number> = {
'2160p': 3840,
'1080p': 1920,
'720p': 1280,
'540p': 960,
'360p': 640,
'180p': 320,
}
export function resolutionToDimensions(
resolution: string,
originalWidth: number,
originalHeight: number,
): [number, number] {
const targetLongSide = RESOLUTION_LONG_SIDES[resolution]
if (!targetLongSide) return [0, 0]
let width: number
let height: number
if (originalWidth >= originalHeight) {
width = targetLongSide
height = Math.round(width * (originalHeight / originalWidth))
} else {
height = targetLongSide
width = Math.round(height * (originalWidth / originalHeight))
}
if (width % 2 !== 0) width++
if (height % 2 !== 0) height++
return [width, height]
}
const TARGET_RESOLUTIONS = [
{ name: '2160p', longSide: 3840 },
{ name: '1080p', longSide: 1920 },
{ name: '720p', longSide: 1280 },
{ name: '540p', longSide: 960 },
{ name: '360p', longSide: 640 },
]
function getBestMatchResolution(
originalWidth: number,
originalHeight: number,
): { name: string; longSide: number } {
const rawLongSide = Math.max(originalWidth, originalHeight)
const lower = TARGET_RESOLUTIONS.filter((r) => r.longSide <= rawLongSide)
if (lower.length > 0) {
lower.sort((a, b) => b.longSide - a.longSide)
return lower[0]
}
return TARGET_RESOLUTIONS[TARGET_RESOLUTIONS.length - 1]
}
export function getTargetVideoResolutions(
strategy: PrismaJson.VideoTranscodeStrategy,
originalWidth: number,
originalHeight: number,
): string[] {
const resolutions = ['180p']
let normalizedStrategy: string = strategy
const stratStr = strategy as string
if (stratStr === 'single' || stratStr === 'disable') {
normalizedStrategy = 'best_match'
} else if (stratStr === 'full') {
normalizedStrategy = 'all'
}
const bestMatch = getBestMatchResolution(originalWidth, originalHeight)
if (normalizedStrategy === 'best_match') {
resolutions.push(bestMatch.name)
} else if (normalizedStrategy === 'all') {
const bestMatchIndex = TARGET_RESOLUTIONS.findIndex((r) => r.name === bestMatch.name)
if (bestMatchIndex !== -1) {
for (let i = bestMatchIndex; i < TARGET_RESOLUTIONS.length; i++) {
resolutions.push(TARGET_RESOLUTIONS[i].name)
}
}
}
return resolutions
}
@@ -0,0 +1,243 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { transcodeVideoWorkflow } from './transcode-video'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
sleep: vi.fn(),
}
})
describe('transcodeVideoWorkflow', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
}),
updateAssetStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getMediaInfoActivity: Object.assign(vi.fn(), { _activityName: 'getMediaInfoActivity' }),
transcodeVideoActivity: Object.assign(vi.fn(), { _activityName: 'transcodeVideoActivity' }),
transcodeAudioActivity: Object.assign(vi.fn(), { _activityName: 'transcodeAudioActivity' }),
transcodeImageActivity: Object.assign(vi.fn(), { _activityName: 'transcodeImageActivity' }),
generateSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generateSpriteActivity' }),
updateAssetMediaActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetMediaActivity',
}),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
downloadMediaToTmpActivity: Object.assign(vi.fn(), {
_activityName: 'downloadMediaToTmpActivity',
}),
cleanupTmpDirActivity: Object.assign(vi.fn(), { _activityName: 'cleanupTmpDirActivity' }),
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
}),
}
beforeEach(() => {
vi.clearAllMocks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(workflowUtils.executeActivity as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(_queue: string, fn: any, ...args: any[]) => {
if (typeof fn !== 'function') {
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
}
return fn(...args)
},
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/video.mp4',
tmpDir: '/tmp',
})
})
it('should process video transcode successfully', async () => {
const task: WorkflowTask = {
id: 'task-1',
assetId: 'asset-1',
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {
videoStrategy: 'best_match',
thumbnail: true,
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-1',
storageKey: { key: 'video.mp4' },
mediaType: 'video/mp4',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
proxyType: 'video',
metadata: {
originalWidth: 1920,
originalHeight: 1080,
duration: 10,
frameRate: 30,
totalFrames: 300,
startTimecode: '00:00:00:00',
bitRate: 1000,
hasAudio: false,
format: {},
},
videoTranscodes: [],
imageTranscodes: [],
})
mockActivities.transcodeVideoActivity.mockResolvedValue({
key: 'v.mp4',
width: 1920,
height: 1080,
})
mockActivities.transcodeImageActivity.mockResolvedValue({
key: 't.webp',
width: 480,
height: 270,
})
await transcodeVideoWorkflow(task)
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
status: AssetStatus.processing,
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
mediaInfo: expect.objectContaining({
thumbnail: expect.objectContaining({
key: 't.webp',
}),
}),
})
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
status: AssetStatus.processed,
})
expect(mockActivities.createEmbeddingTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
teamId: 'team-1',
projectId: 'proj-1',
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-1',
status: WorkflowTaskStatus.completed,
})
})
it('should run audio transcoding workflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-audio',
assetId: 'asset-audio',
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid-2',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-audio',
storageKey: { key: 'audio.wav' },
mediaType: 'audio/wav',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
proxyType: 'audio',
metadata: {
originalWidth: 0,
originalHeight: 0,
duration: 30,
bitRate: 1411200,
frameRate: 0,
totalFrames: 0,
hasAudio: true,
audioCodec: 'pcm_s16le',
audioChannels: 2,
},
original: {
key: 'audio.wav',
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
},
videoTranscodes: [],
})
mockActivities.transcodeAudioActivity.mockResolvedValue({
width: 0,
height: 0,
key: 'files/asset-audio/audio-audio-proxy.mp4',
})
await transcodeVideoWorkflow(task)
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith({
assetKey: 'audio.wav',
filePath: '/tmp/video.mp4',
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-audio',
mediaInfo: expect.objectContaining({
proxyType: 'audio',
videoTranscodes: [
{
width: 0,
height: 0,
key: 'files/asset-audio/audio-audio-proxy.mp4',
},
],
}),
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-audio',
status: WorkflowTaskStatus.completed,
})
})
})
@@ -0,0 +1,172 @@
import type { WorkflowTask } from '@shumai/db'
import '@shumai/db/src/prisma-json-types'
import { executeActivity, getActivities } from '@shumai/workflow-core'
import {
getWorkerQueueAndStartTask,
fetchAssetWithKey,
completeTask,
failTask,
cleanupTmpDir,
} from './common'
import { getTargetVideoResolutions, resolutionToDimensions } from './transcode-utils'
export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void> {
let tmpDir: string | undefined
let workerQueue = ''
try {
workerQueue = await getWorkerQueueAndStartTask(task)
const {
updateAssetStatusActivity,
getMediaInfoActivity,
transcodeVideoActivity,
transcodeAudioActivity,
transcodeImageActivity,
generateSpriteActivity,
updateAssetMediaActivity,
downloadMediaToTmpActivity,
createEmbeddingTaskIfEnabledActivity,
} = getActivities()
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: task.assetId,
status: 'processing',
})
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, {
assetKey: key,
})
const { filePath } = download
tmpDir = download.tmpDir
const spec = task.payload?.transcode || {}
const proxyType = asset.mediaType?.startsWith('audio/') ? 'audio' : 'video'
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath,
assetId: asset.id,
proxyType,
mediaType: asset.mediaType || '',
})
mediaInfo.original = {
key,
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
}
const metadata = mediaInfo.metadata
if (mediaInfo.proxyType === 'video' && metadata) {
const videoResolutions = getTargetVideoResolutions(
spec.videoStrategy || 'best_match',
metadata.originalWidth,
metadata.originalHeight,
)
for (const res of videoResolutions) {
const [width, height] = resolutionToDimensions(
res,
metadata.originalWidth,
metadata.originalHeight,
)
const videoSpec: PrismaJson.VideoTranscode = {
resolution: res,
width,
height,
}
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
assetKey: key,
filePath,
videoSpec,
duration: metadata.duration,
originalFps: metadata.frameRate,
})
if (res === '180p') {
mediaInfo.videoPreview = videoTranscode
} else {
mediaInfo.videoTranscodes.push(videoTranscode)
}
}
mediaInfo.videoTranscodes.push({
key: mediaInfo.original?.key,
width: metadata.originalWidth,
height: metadata.originalHeight,
isRaw: true,
})
}
const isAudio = mediaInfo.proxyType === 'audio'
if (isAudio && metadata) {
const audioTranscode = await executeActivity(workerQueue, transcodeAudioActivity, {
assetKey: key,
filePath,
})
mediaInfo.videoTranscodes.push(audioTranscode)
}
if (spec.thumbnail) {
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: key,
filePath,
imageSpec: { width: 480, height: 0, quality: 80, format: 'webp' },
})
mediaInfo.thumbnail = thumbTranscode
}
if (spec.sprite || spec.poster) {
const lastSlashIndex = key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
const spriteSpec: PrismaJson.SpriteInfo = {
key: assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp',
frames: 100,
tileX: 10,
tileY: 10,
}
const posterSpec: PrismaJson.PosterInfo = {
key: assetDir ? `${assetDir}/poster.webp` : 'poster.webp',
}
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
assetKey: key,
filePath,
spriteSpec,
posterSpec,
mediaInfo,
})
mediaInfo.sprite = spriteResult.sprite
mediaInfo.poster = spriteResult.poster
}
await executeActivity(workerQueue, updateAssetMediaActivity, {
assetId: asset.id,
mediaInfo,
})
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: asset.id,
status: 'processed',
})
await executeActivity(workerQueue, createEmbeddingTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
await completeTask(workerQueue, task.id)
} catch (err) {
console.error(`transcodeVideoWorkflow failed for task ${task.id}:`, err)
await failTask(workerQueue, task.id, err)
throw err
} finally {
await cleanupTmpDir(workerQueue, tmpDir)
}
}
@@ -1,6 +1,6 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { transcodeMedia } from './transcode'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async (importOriginal) => {
@@ -14,7 +14,7 @@ vi.mock('@shumai/workflow-core', async (importOriginal) => {
}
})
describe('Transcode Workflow', () => {
describe('transcodeMedia Fallback Dispatcher', () => {
const mockActivities = {
updateTaskStatusActivity: Object.assign(vi.fn(), {
_activityName: 'updateTaskStatusActivity',
@@ -23,27 +23,11 @@ describe('Transcode Workflow', () => {
_activityName: 'updateAssetStatusActivity',
}),
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
getMediaInfoActivity: Object.assign(vi.fn(), { _activityName: 'getMediaInfoActivity' }),
transcodeVideoActivity: Object.assign(vi.fn(), { _activityName: 'transcodeVideoActivity' }),
transcodeAudioActivity: Object.assign(vi.fn(), { _activityName: 'transcodeAudioActivity' }),
transcodeImageActivity: Object.assign(vi.fn(), { _activityName: 'transcodeImageActivity' }),
generateSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generateSpriteActivity' }),
updateAssetMediaActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetMediaActivity',
}),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
downloadMediaToTmpActivity: Object.assign(vi.fn(), {
_activityName: 'downloadMediaToTmpActivity',
}),
cleanupTmpDirActivity: Object.assign(vi.fn(), { _activityName: 'cleanupTmpDirActivity' }),
takeScreenshotsActivity: Object.assign(vi.fn(), { _activityName: 'takeScreenshotsActivity' }),
overlayAnnotationsActivity: Object.assign(vi.fn(), {
_activityName: 'overlayAnnotationsActivity',
}),
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
renderPdfPagesActivity: Object.assign(vi.fn(), {
_activityName: 'renderPdfPagesActivity',
}),
}
@@ -63,27 +47,23 @@ describe('Transcode Workflow', () => {
)
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/video.mp4',
tmpDir: '/tmp',
})
})
it('should process video transcode successfully', async () => {
it('should dispatch to pdfPages workflow when payload.pdfPages is present', async () => {
const task: WorkflowTask = {
id: 'task-1',
assetId: 'asset-1',
id: 'task-pdf-pages',
assetId: 'asset-pdf',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {
videoStrategy: 'best_match',
thumbnail: true,
pdfPages: {
start: 1,
end: 2,
},
},
output: null,
sessionId: null,
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
@@ -96,153 +76,16 @@ describe('Transcode Workflow', () => {
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-1',
storageKey: { key: 'video.mp4' },
mediaType: 'video/mp4',
id: 'asset-pdf',
storageKey: { key: 'doc.pdf' },
mediaType: 'application/pdf',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
mimeType: 'video/mp4',
metadata: {
originalWidth: 1920,
originalHeight: 1080,
duration: 10,
frameRate: 30,
totalFrames: 300,
startTimecode: '00:00:00:00',
bitRate: 1000,
hasAudio: false,
format: {},
},
videoTranscodes: [],
imageTranscodes: [],
})
mockActivities.transcodeVideoActivity.mockResolvedValue({
key: 'v.mp4',
width: 1920,
height: 1080,
})
mockActivities.transcodeImageActivity.mockResolvedValue({
key: 't.webp',
width: 480,
height: 270,
})
mockActivities.renderPdfPagesActivity.mockResolvedValue([])
await transcodeMedia(task)
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
status: AssetStatus.processing,
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
mediaInfo: expect.objectContaining({
thumbnail: expect.objectContaining({
key: 't.webp',
}),
}),
})
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
status: AssetStatus.processed,
})
expect(mockActivities.createEmbeddingTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
teamId: 'team-1',
projectId: 'proj-1',
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-1',
status: WorkflowTaskStatus.completed,
})
})
it('should process image transcode and thumbnail successfully', async () => {
const task: WorkflowTask = {
id: 'task-image',
assetId: 'asset-image',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {
thumbnail: true,
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-image',
storageKey: { key: 'image.jpg' },
mediaType: 'image/jpeg',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
mimeType: 'image/jpeg',
metadata: {
originalWidth: 1000,
originalHeight: 1000,
duration: 0,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
bitRate: 0,
hasAudio: false,
format: {},
},
videoTranscodes: [],
imageTranscodes: [],
})
mockActivities.transcodeImageActivity.mockResolvedValue({
key: 't.webp',
width: 480,
height: 480,
format: 'webp',
})
await transcodeMedia(task)
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
status: AssetStatus.processing,
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
mediaInfo: expect.objectContaining({
thumbnail: expect.objectContaining({
key: 't.webp',
width: 480,
height: 480,
}),
}),
})
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
status: AssetStatus.processed,
})
expect(mockActivities.createEmbeddingTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
teamId: 'team-1',
projectId: 'proj-1',
})
expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalled()
})
it('should handle failures and update task status with error', async () => {
@@ -277,205 +120,4 @@ describe('Transcode Workflow', () => {
output: { error: 'FFmpeg failed' },
})
})
it('should run screenshot workflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-screenshot',
assetId: 'asset-video',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
screenshot: {
start: 0,
end: 10,
count: 2,
commentTimestamp: 5.0,
annotations: [],
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-video',
storageKey: { key: 'video.mp4' },
mediaType: 'video/mp4',
})
mockActivities.takeScreenshotsActivity.mockResolvedValue([
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
])
await transcodeMedia(task)
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
assetKey: 'video.mp4',
assetId: 'asset-video',
start: 0,
end: 10,
count: 2,
commentTimestamp: 5.0,
annotations: [],
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-screenshot',
status: WorkflowTaskStatus.completed,
output: {
screenshots: [
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
],
},
})
})
it('should run image annotation overlay workflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-image-ann',
assetId: 'asset-image',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
imageAnnotation: {
annotations: [{ type: 'box', color: '#ff0000', points: [] }],
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-image',
storageKey: { key: 'image.png' },
mediaType: 'image/png',
media: {
imageTranscodes: [{ key: 'image-transcoded.webp' }],
},
})
mockActivities.overlayAnnotationsActivity.mockResolvedValue(
'files/asset-image/annotations/ann1.webp',
)
await transcodeMedia(task)
expect(mockActivities.overlayAnnotationsActivity).toHaveBeenCalledWith({
assetKey: 'image-transcoded.webp',
assetId: 'asset-image',
annotations: [{ type: 'box', color: '#ff0000', points: [] }],
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-image-ann',
status: WorkflowTaskStatus.completed,
output: { key: 'files/asset-image/annotations/ann1.webp' },
})
})
it('should run audio transcoding workflow successfully', async () => {
const task: WorkflowTask = {
id: 'task-audio',
assetId: 'asset-audio',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid-2',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-audio',
storageKey: { key: 'audio.wav' },
mediaType: 'audio/wav',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
mimeType: 'audio/wav',
metadata: {
originalWidth: 0,
originalHeight: 0,
duration: 30,
bitRate: 1411200,
frameRate: 0,
totalFrames: 0,
hasAudio: true,
audioCodec: 'pcm_s16le',
audioChannels: 2,
},
original: {
key: 'audio.wav',
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
},
videoTranscodes: [],
})
mockActivities.transcodeAudioActivity.mockResolvedValue({
width: 0,
height: 0,
key: 'files/asset-audio/audio-audio-proxy.mp4',
})
await transcodeMedia(task)
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith({
assetKey: 'audio.wav',
filePath: '/tmp/video.mp4',
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-audio',
mediaInfo: expect.objectContaining({
mimeType: 'audio/wav',
videoTranscodes: [
{
width: 0,
height: 0,
key: 'files/asset-audio/audio-audio-proxy.mp4',
},
],
}),
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-audio',
status: WorkflowTaskStatus.completed,
})
})
})
+39 -388
View File
@@ -1,395 +1,46 @@
import type { WorkflowTask } from '@shumai/db'
import { executeActivity, getActivities, TaskQueueTranscode } from '@shumai/workflow-core'
import { ApplicationFailure } from '@temporalio/workflow'
import { transcodeVideoWorkflow } from './transcode-video'
import { transcodeImageWorkflow } from './transcode-image'
import { transcodePdfWorkflow } from './transcode-pdf'
import { renderPdfPagesWorkflow } from './render-pdf-pages'
import { takeVideoScreenshotsWorkflow } from './take-video-screenshots'
import { overlayImageAnnotationWorkflow } from './overlay-image-annotation'
/**
* @deprecated Legacy monolithic transcode workflow. Prefer using focused workflows:
* `transcodeVideoWorkflow`, `transcodeImageWorkflow`, `transcodePdfWorkflow`,
* `renderPdfPagesWorkflow`, `takeVideoScreenshotsWorkflow`, or `overlayImageAnnotationWorkflow`.
*/
export async function transcodeMedia(task: WorkflowTask): Promise<void> {
const {
updateTaskStatusActivity,
updateAssetStatusActivity,
getAssetActivity,
getMediaInfoActivity,
transcodeVideoActivity,
transcodeAudioActivity,
transcodeImageActivity,
generateSpriteActivity,
updateAssetMediaActivity,
getTranscodeWorkerQueueActivity,
downloadMediaToTmpActivity,
cleanupTmpDirActivity,
takeScreenshotsActivity,
overlayAnnotationsActivity,
createEmbeddingTaskIfEnabledActivity,
} = getActivities()
let tmpDir: string | undefined
let workerQueue = ''
try {
// 0. Get Worker-Specific Queue for Transcode
workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity)
// 1. Update status to processing
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId: task.id,
status: 'processing',
})
const payload = task.payload
if (payload?.screenshot) {
const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId)
const key = asset?.storageKey?.key
if (!asset || !key) {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
nonRetryable: true,
})
}
const screenshots = await executeActivity(workerQueue, takeScreenshotsActivity, {
assetKey: key,
assetId: asset.id,
start: payload.screenshot.start,
end: payload.screenshot.end,
count: payload.screenshot.count,
commentTimestamp: payload.screenshot.commentTimestamp,
annotations: payload.screenshot.annotations,
})
// Update Task Status to completed with output
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId: task.id,
status: 'completed',
output: { screenshots },
})
return
}
if (payload?.imageAnnotation) {
const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId)
let key = asset?.storageKey?.key
if (asset?.media) {
const mediaInfo = asset.media as unknown as PrismaJson.MediaInfo
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
key = mediaInfo.imageTranscodes[0].key || key
}
}
if (!asset || !key) {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
nonRetryable: true,
})
}
const keyWithOverlay = await executeActivity(workerQueue, overlayAnnotationsActivity, {
assetKey: key,
assetId: asset.id,
annotations: payload.imageAnnotation.annotations,
})
// Update Task Status to completed with output
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId: task.id,
status: 'completed',
output: { key: keyWithOverlay },
})
return
}
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: task.assetId,
status: 'processing',
})
// 2. Get Asset
const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId)
const key = asset?.storageKey?.key
if (!asset || !key) {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
nonRetryable: true,
})
}
// 4. Download Media to Tmp
const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, {
assetKey: key,
})
const { filePath } = download
tmpDir = download.tmpDir
// 5. Get Media Info
const spec = task.payload?.transcode || {}
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath,
assetId: asset.id,
mediaType: asset.mediaType || '',
})
mediaInfo.original = {
key,
downloadUrl: '',
filesizeInBytes: 0,
codec: '',
}
const mimeType = mediaInfo.mimeType
const metadata = mediaInfo.metadata
// 6. Video Transcoding
if (mimeType.startsWith('video/') && metadata) {
const videoResolutions = getTargetVideoResolutions(
spec.videoStrategy || 'best_match',
metadata.originalWidth,
metadata.originalHeight,
)
for (const res of videoResolutions) {
const [width, height] = resolutionToDimensions(
res,
metadata.originalWidth,
metadata.originalHeight,
)
const videoSpec: PrismaJson.VideoTranscode = {
resolution: res,
width,
height,
}
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
assetKey: key,
filePath,
videoSpec,
duration: metadata.duration,
originalFps: metadata.frameRate,
})
if (res === '180p') {
mediaInfo.videoPreview = videoTranscode
} else {
mediaInfo.videoTranscodes.push(videoTranscode)
}
}
// Always keep raw
mediaInfo.videoTranscodes.push({
key: mediaInfo.original?.key,
width: metadata.originalWidth,
height: metadata.originalHeight,
isRaw: true,
})
}
// 6.5 Audio Transcoding
const isAudio = mimeType.startsWith('audio/')
if (isAudio && metadata) {
const audioTranscode = await executeActivity(workerQueue, transcodeAudioActivity, {
assetKey: key,
filePath,
})
mediaInfo.videoTranscodes.push(audioTranscode)
}
// 7. Image Transcoding
const isImage = mimeType.startsWith('image/')
const isPsd = isMimePsd(mimeType)
if ((isImage || isPsd) && metadata) {
const imageSpec: PrismaJson.ImageTranscode = {
width: metadata.originalWidth,
height: metadata.originalHeight,
quality: 90,
format: 'webp',
}
const imageTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: key,
filePath,
imageSpec,
})
mediaInfo.imageTranscodes.push(imageTranscode)
}
// 9. System: Thumbnail
if (spec.thumbnail) {
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: key,
filePath,
imageSpec: { width: 480, height: 0, quality: 80, format: 'webp' },
})
mediaInfo.thumbnail = thumbTranscode
}
// 10. System: Sprite/Poster (Video only usually)
if (spec.sprite || spec.poster) {
const lastSlashIndex = key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
const spriteSpec: PrismaJson.SpriteInfo = {
key: assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp',
frames: 100,
tileX: 10,
tileY: 10,
}
const posterSpec: PrismaJson.PosterInfo = {
key: assetDir ? `${assetDir}/poster.webp` : 'poster.webp',
}
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
assetKey: key,
filePath,
spriteSpec,
posterSpec,
mediaInfo,
})
mediaInfo.sprite = spriteResult.sprite
mediaInfo.poster = spriteResult.poster
}
// 11. Update Asset Media and Status
await executeActivity(workerQueue, updateAssetMediaActivity, {
assetId: asset.id,
mediaInfo,
})
await executeActivity(workerQueue, updateAssetStatusActivity, {
assetId: asset.id,
status: 'processed',
})
// 11.5 Create AI embedding task if enabled
await executeActivity(workerQueue, createEmbeddingTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
// 12. Update Task Status
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId: task.id,
status: 'completed',
})
} catch (err) {
console.error(`TranscodeMedia failed for task ${task.id}:`, err)
// Update status to failed
if (workerQueue) {
await executeActivity(workerQueue, updateTaskStatusActivity, {
taskId: task.id,
status: 'failed',
output: { error: err instanceof Error ? err.message : String(err) },
})
}
throw err
} finally {
if (tmpDir && workerQueue) {
try {
await executeActivity(workerQueue, cleanupTmpDirActivity, { tmpDir })
} catch (cleanupErr) {
console.error('Failed to cleanup tmp dir:', cleanupErr)
}
}
const payload = task.payload
if (payload?.pdfPages) {
return renderPdfPagesWorkflow(task)
}
if (payload?.screenshot) {
return takeVideoScreenshotsWorkflow(task)
}
if (payload?.imageAnnotation) {
return overlayImageAnnotationWorkflow(task)
}
if (task.type === 'transcode_image') {
return transcodeImageWorkflow(task)
}
if (task.type === 'transcode_pdf') {
return transcodePdfWorkflow(task)
}
return transcodeVideoWorkflow(task)
}
function isMimePsd(mimeType: string): boolean {
switch (mimeType) {
case 'image/vnd.adobe.photoshop':
case 'image/x-photoshop':
case 'application/x-photoshop':
case 'image/psd':
return true
}
return false
}
const RESOLUTION_LONG_SIDES: Record<string, number> = {
'2160p': 3840,
'1080p': 1920,
'720p': 1280,
'540p': 960,
'360p': 640,
'180p': 320,
}
function resolutionToDimensions(
resolution: string,
originalWidth: number,
originalHeight: number,
): [number, number] {
const targetLongSide = RESOLUTION_LONG_SIDES[resolution]
if (!targetLongSide) return [0, 0]
let width: number
let height: number
if (originalWidth >= originalHeight) {
// Landscape or square: width is the long side
width = targetLongSide
height = Math.round(width * (originalHeight / originalWidth))
} else {
// Portrait: height is the long side
height = targetLongSide
width = Math.round(height * (originalWidth / originalHeight))
}
// Ensure even dimensions for H.264
if (width % 2 !== 0) width++
if (height % 2 !== 0) height++
return [width, height]
}
const TARGET_RESOLUTIONS = [
{ name: '2160p', longSide: 3840 },
{ name: '1080p', longSide: 1920 },
{ name: '720p', longSide: 1280 },
{ name: '540p', longSide: 960 },
{ name: '360p', longSide: 640 },
]
function getBestMatchResolution(
originalWidth: number,
originalHeight: number,
): { name: string; longSide: number } {
const rawLongSide = Math.max(originalWidth, originalHeight)
const lower = TARGET_RESOLUTIONS.filter((r) => r.longSide <= rawLongSide)
if (lower.length > 0) {
lower.sort((a, b) => b.longSide - a.longSide)
return lower[0]
}
return TARGET_RESOLUTIONS[TARGET_RESOLUTIONS.length - 1] // 360p (longSide 640)
}
function getTargetVideoResolutions(
strategy: PrismaJson.VideoTranscodeStrategy,
originalWidth: number,
originalHeight: number,
): string[] {
const resolutions = ['180p']
// Normalize legacy strategies for backward compatibility
let normalizedStrategy: string = strategy
const stratStr = strategy as string
if (stratStr === 'single' || stratStr === 'disable') {
normalizedStrategy = 'best_match'
} else if (stratStr === 'full') {
normalizedStrategy = 'all'
}
const bestMatch = getBestMatchResolution(originalWidth, originalHeight)
if (normalizedStrategy === 'best_match') {
resolutions.push(bestMatch.name)
} else if (normalizedStrategy === 'all') {
const bestMatchIndex = TARGET_RESOLUTIONS.findIndex((r) => r.name === bestMatch.name)
if (bestMatchIndex !== -1) {
for (let i = bestMatchIndex; i < TARGET_RESOLUTIONS.length; i++) {
resolutions.push(TARGET_RESOLUTIONS[i].name)
}
}
}
return resolutions
}
/**
* @deprecated Legacy alias for transcodeMedia workflow.
*/
export const transcodeWorkflow = transcodeMedia
export * from './transcode-video'
export * from './transcode-image'
export * from './transcode-pdf'
export * from './render-pdf-pages'
export * from './take-video-screenshots'
export * from './overlay-image-annotation'
export * from './common'
export * from './transcode-utils'
+2 -2
View File
@@ -50,7 +50,7 @@ interface BreadcrumbNavProps {
name?: string
type: 'file' | 'folder'
version?: number
mediaType?: string | null
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
}
isRootFolder: boolean
displayStyle?: 'card' | 'list'
@@ -158,7 +158,7 @@ export function BreadcrumbNav({
}
}
const isAudio = currentAsset.mediaType?.startsWith('audio/')
const isAudio = currentAsset.proxyType === 'audio'
const hasVideoTranscodes =
!isAudio &&
downloadInfo?.videoTranscodes &&
@@ -51,6 +51,7 @@ interface MessageCardProps {
onSelect?: () => void
frameRate?: number
startTimecode?: string
formatTimestamp?: (second: number) => string
rootParentId?: string
}
@@ -111,6 +112,7 @@ export const MessageCard: React.FC<MessageCardProps> = ({
onSelect,
frameRate,
startTimecode,
formatTimestamp,
}) => {
const message = initialMessage
const hasDrawInfo =
@@ -124,10 +126,14 @@ export const MessageCard: React.FC<MessageCardProps> = ({
const { videoTimeDisplayMode } = useUiStore()
const displayTime = React.useMemo(() => {
if (message.second === null || message.second === undefined || !frameRate) return ''
if (message.second === null || message.second === undefined) return ''
if (formatTimestamp) {
return formatTimestamp(message.second)
}
if (!frameRate) return ''
const frameIndex = Math.round(message.second * frameRate)
return formatTimecode(frameIndex, frameRate, videoTimeDisplayMode, startTimecode)
}, [message.second, frameRate, videoTimeDisplayMode, startTimecode])
}, [message.second, frameRate, videoTimeDisplayMode, startTimecode, formatTimestamp])
const { data: me } = useQuery({
queryKey: ['teams', teamId, 'me'],
@@ -340,7 +346,7 @@ export const MessageCard: React.FC<MessageCardProps> = ({
{message.attachments && message.attachments.length > 0 && (
<div className="flex flex-wrap gap-2 mt-3">
{message.attachments.map((att) => {
const isImage = att.mediaType?.startsWith('image/')
const isImage = att.proxyType === 'image'
const name = att.url?.split('/').pop()?.split('?')[0] || 'file'
return (
<div
@@ -57,6 +57,7 @@ interface ChatInputProps {
currentTime?: number
frameRate?: number
startTimecode?: string
formatTimestamp?: (second: number) => string
onTyping?: () => void
}
@@ -88,6 +89,7 @@ export const ChatInput = React.forwardRef<HTMLDivElement, ChatInputProps>(
currentTime,
frameRate,
startTimecode,
formatTimestamp,
onTyping,
},
ref,
@@ -98,10 +100,14 @@ export const ChatInput = React.forwardRef<HTMLDivElement, ChatInputProps>(
const { videoTimeDisplayMode } = useUiStore()
const displayTime = React.useMemo(() => {
if (currentTime === undefined || !frameRate) return ''
if (currentTime === undefined) return ''
if (formatTimestamp) {
return formatTimestamp(currentTime)
}
if (!frameRate) return ''
const frameIndex = Math.round(currentTime * frameRate)
return formatTimecode(frameIndex, frameRate, videoTimeDisplayMode, startTimecode)
}, [currentTime, frameRate, videoTimeDisplayMode, startTimecode])
}, [currentTime, frameRate, videoTimeDisplayMode, startTimecode, formatTimestamp])
const { teamId, ensureTeamIdForProject } = useTeamContextStore()
const { members: storeMembers, loading: membersLoading, fetchMembers } = useMemberStore()
@@ -135,7 +141,12 @@ export const ChatInput = React.forwardRef<HTMLDivElement, ChatInputProps>(
const [hasContent, setHasContent] = useState(false)
const paddingLeftClass = React.useMemo(() => {
if (!isTimestampEnabled || currentTime === undefined || !frameRate || hasContent) {
if (
!isTimestampEnabled ||
currentTime === undefined ||
(!formatTimestamp && !frameRate) ||
hasContent
) {
return ''
}
if (videoTimeDisplayMode === 'timecode') {
@@ -4,19 +4,19 @@ import { kindOf, pickDefaultCompareVersions, clampFrame, isSameType } from './co
describe('compare-utils', () => {
describe('kindOf', () => {
it('classifies image media types', () => {
expect(kindOf('image/png')).toBe('image')
expect(kindOf('image/webp')).toBe('image')
expect(kindOf('image')).toBe('image')
expect(kindOf({ proxyType: 'image' })).toBe('image')
})
it('classifies video media types', () => {
expect(kindOf('video/mp4')).toBe('video')
expect(kindOf('video/quicktime')).toBe('video')
expect(kindOf('video')).toBe('video')
expect(kindOf({ proxyType: 'video' })).toBe('video')
})
it('returns unsupported for missing or unknown types', () => {
expect(kindOf(null)).toBe('unsupported')
expect(kindOf(undefined)).toBe('unsupported')
expect(kindOf('application/pdf')).toBe('unsupported')
expect(kindOf('pdf')).toBe('unsupported')
})
})
@@ -1,10 +1,11 @@
import type { PaneKind } from './types'
/** Classify an asset's media type into a compare pane kind. */
export function kindOf(mediaType: string | null | undefined): PaneKind {
if (!mediaType) return 'unsupported'
if (mediaType.startsWith('image/')) return 'image'
if (mediaType.startsWith('video/')) return 'video'
export function kindOf(item: { proxyType?: string | null } | string | null | undefined): PaneKind {
if (!item) return 'unsupported'
const proxyType = typeof item === 'object' ? item.proxyType : item
if (proxyType === 'image') return 'image'
if (proxyType === 'video') return 'video'
return 'unsupported'
}
@@ -513,10 +513,22 @@ export function FileBrowser({
fileCount: 0,
type: 'file',
status: hasUrl ? 'uploading' : 'error',
mediaType:
proxyType:
f.file.name.toLowerCase().endsWith('.wma') && f.file.type?.startsWith('video/')
? f.file.type.replace(/^video\//, 'audio/')
: f.file.type || null,
? 'audio'
: f.file.type?.startsWith('image/')
? 'image'
: f.file.type?.startsWith('video/')
? 'video'
: f.file.type?.startsWith('audio/')
? 'audio'
: f.file.name.endsWith('.pdf') ||
f.file.name.endsWith('.txt') ||
f.file.name.endsWith('.csv') ||
f.file.name.endsWith('.md') ||
f.file.name.endsWith('.markdown')
? 'pdf'
: null,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
}
@@ -214,8 +214,7 @@ export function FileListItem({
alt=""
className="w-full h-full object-cover"
/>
) : displayItem.preview?.mediaType?.startsWith('audio/') ||
displayItem.mediaType?.startsWith('audio/') ? (
) : displayItem.preview?.proxyType === 'audio' || displayItem.proxyType === 'audio' ? (
<AudioLines className="h-4 w-4 text-muted-foreground" />
) : displayItem.type === 'folder' ? (
<Folder className="h-4 w-4 text-primary" />
@@ -4,13 +4,16 @@ import { formatTime } from '../viewers/video/utils'
type FilePreviewItem = {
type?: string | null
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
preview?: {
mediaType?: string | null
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
spriteUrl?: string
thumbnailUrl?: string
originalWidth?: number
originalHeight?: number
duration?: number
pageCount?: number
} | null
}
@@ -20,25 +23,33 @@ interface FilePreviewProps {
}
export const FilePreview = ({ item, showDuration = false }: FilePreviewProps) => {
const isVideo = item.preview?.mediaType?.startsWith('video/')
const isAudio = item.preview?.mediaType?.startsWith('audio/')
const proxyType = item.preview?.proxyType || item.proxyType
const isVideo = proxyType === 'video'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
const hasDuration = isVideo || isAudio
const duration = item.preview?.duration
const durationOverlay =
const pageCount = item.preview?.pageCount
const overlay =
showDuration && hasDuration && typeof duration === 'number' && duration > 0 ? (
<span className="pointer-events-none absolute bottom-1 right-1 text-xs font-medium tabular-nums text-white">
<span className="pointer-events-none absolute bottom-1 right-1 rounded bg-black/60 px-1 py-0.5 text-xs font-medium tabular-nums text-white">
{formatTime(duration)}
</span>
) : showDuration && isPdf && typeof pageCount === 'number' && pageCount > 0 ? (
<span className="pointer-events-none absolute bottom-1 right-1 rounded bg-black/60 px-1 py-0.5 text-xs font-medium tabular-nums text-white">
{pageCount} P
</span>
) : null
const isVideoWithSprite =
isVideo &&
const isMediaWithSprite =
(isVideo || isPdf) &&
item.preview?.spriteUrl &&
item.preview.thumbnailUrl &&
item.preview.originalWidth &&
item.preview.originalHeight
if (isVideoWithSprite && item.preview) {
if (isMediaWithSprite && item.preview) {
return (
<>
<SpriteScrubber
@@ -47,7 +58,7 @@ export const FilePreview = ({ item, showDuration = false }: FilePreviewProps) =>
videoWidth={item.preview.originalWidth!}
videoHeight={item.preview.originalHeight!}
/>
{durationOverlay}
{overlay}
</>
)
}
@@ -60,7 +71,7 @@ export const FilePreview = ({ item, showDuration = false }: FilePreviewProps) =>
alt="Preview"
className="w-full h-full object-contain bg-black"
/>
{durationOverlay}
{overlay}
</>
)
}
@@ -71,7 +82,7 @@ export const FilePreview = ({ item, showDuration = false }: FilePreviewProps) =>
<div className="w-full h-full flex items-center justify-center bg-zinc-950">
<AudioLines className="w-8 h-8 text-zinc-500" />
</div>
{durationOverlay}
{overlay}
</>
)
}

Some files were not shown because too many files have changed in this diff Show More