mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-20 10:57:40 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6afcc135e | ||
|
|
fb28b7d020 | ||
|
|
28c3df3429 | ||
|
|
93e957fa4f | ||
|
|
1ac0d1452e | ||
|
|
5855134236 | ||
|
|
581a12bdc9 | ||
|
|
851e5d975c | ||
|
|
6e69fc451b | ||
|
|
b91033495c | ||
|
|
a2895cd5d1 | ||
|
|
e3352232fe | ||
|
|
6d34b13266 | ||
|
|
208eb6898a | ||
|
|
b1d9c1b31b | ||
|
|
c016d1dffe | ||
|
|
7cd8a5bef7 | ||
|
|
3274d19574 | ||
|
|
047174c13d | ||
|
|
d9d7b52758 | ||
|
|
52edcd1a79 | ||
|
|
3afe0fc3c3 | ||
|
|
5d4020dc2d | ||
|
|
4ad0df5876 | ||
|
|
fea153e94f | ||
|
|
f158d04686 | ||
|
|
c45a1d9ee9 | ||
|
|
29995a64fd | ||
|
|
1970535aa6 | ||
|
|
7574dd56e4 | ||
|
|
081dbf6d39 | ||
|
|
817889e501 | ||
|
|
aa572bf2d6 | ||
|
|
d9ed30fedb | ||
|
|
7d7a1b61c0 | ||
|
|
cbe622dfeb | ||
|
|
b72a9435e7 | ||
|
|
759e37915b | ||
|
|
e2523e1a81 | ||
|
|
6064a1449a | ||
|
|
f68ba70656 | ||
|
|
68a0fb7319 | ||
|
|
661ec06cef | ||
|
|
6000b8e366 | ||
|
|
db3551de1d | ||
|
|
dc20c7f548 | ||
|
|
f76e7a8e8d | ||
|
|
cfdc085ae7 | ||
|
|
779706a3e7 | ||
|
|
06dc929a2a | ||
|
|
0ad3aca083 | ||
|
|
b59f439683 | ||
|
|
d270f3f62c | ||
|
|
ea201ad19f | ||
|
|
2b72f3038a | ||
|
|
c1ddbe4be7 | ||
|
|
3ada194a8e | ||
|
|
bc1491bed5 | ||
|
|
3383079787 | ||
|
|
f888a575b2 | ||
|
|
0a7d5d1f46 | ||
|
|
879922a9fe | ||
|
|
58967d1a63 | ||
|
|
e978878a90 | ||
|
|
ecfbfc2bac | ||
|
|
490c6a3da6 | ||
|
|
62b1fc1d37 | ||
|
|
fa6bf73452 | ||
|
|
25c747aa87 | ||
|
|
ff7712744d | ||
|
|
6f369625d8 | ||
|
|
c6acd075e7 | ||
|
|
7d9f3276f1 | ||
|
|
d524a81ecd | ||
|
|
83c427e646 | ||
|
|
b018fd0fec |
@@ -32,7 +32,7 @@ jobs:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep bubblewrap socat
|
||||
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils
|
||||
|
||||
- name: Lint
|
||||
run: bun run lint
|
||||
@@ -138,3 +138,26 @@ jobs:
|
||||
name: playwright-report-app-${{ matrix.project }}
|
||||
path: playwright-report/
|
||||
retention-days: 7
|
||||
|
||||
workflow_e2e:
|
||||
name: Workflow E2E
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v1
|
||||
with:
|
||||
bun-version: latest
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils
|
||||
|
||||
- name: Run Workflow E2E Tests
|
||||
run: bun run test:e2e:workflow
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep bubblewrap socat libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find
|
||||
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
|
||||
@@ -91,6 +91,7 @@ Add any additional context, caveats, or follow-up work.
|
||||
- `bun run typecheck`
|
||||
- `bun run test`
|
||||
- `bun run test:e2e`
|
||||
- `bun run test:e2e:workflow`
|
||||
|
||||
- **Backend Testing Mandate**: Every backend feature, service method, workflow, and activity MUST be accompanied by comprehensive tests. Logic-heavy code without corresponding test coverage is considered incomplete.
|
||||
|
||||
@@ -140,7 +141,7 @@ We follow Bun's monorepo conventions for dependency management:
|
||||
3. **Local DevDependencies**: Tools specific to a single package (e.g., `@vitejs/plugin-react` for `webui`) should be declared in that package's local `package.json`.
|
||||
4. **Workspace Imports**: Use `workspace:*` for internal package dependencies.
|
||||
5. **Clean Root**: The root `package.json` must not contain runtime `dependencies`. It is reserved for shared `devDependencies` and workspace-wide `scripts`.
|
||||
6. **Installation**: Always run `bun install` from the root. Use the `--filter` flag if you only want to update a specific package (e.g., `bun install --filter @shumai/webui`).
|
||||
6. **Adding Packages**: Never edit `package.json` manually to add new dependencies. Always use `bun add` (e.g., `bun add <package> --filter <workspace>`) from the root without specifying an explicit version, letting Bun resolve the correct version.
|
||||
|
||||
## TypeScript Configuration
|
||||
|
||||
@@ -273,6 +274,14 @@ describe('Team API', () => {
|
||||
})
|
||||
```
|
||||
|
||||
### Workflow E2E Tests
|
||||
|
||||
- Located in `packages/e2e/workflow/**/*.test.ts`.
|
||||
- Run via `bun run test:e2e:workflow` (which executes tests against both `local` and `temporal` executors dynamically).
|
||||
- **Mandatory test coverage**: When creating or updating workflows or activities, you MUST create or update the corresponding E2E tests.
|
||||
- **Mocking**: Mock ONLY AI API calls. Do not mock S3, databases, or media/transcode extraction services.
|
||||
- Mock the AI response by spying on `AgentHarness.prototype.prompt`:
|
||||
|
||||
## Prisma Configuration & Migrations
|
||||
|
||||
- Schema: `packages/db/prisma/schema.prisma`
|
||||
@@ -305,6 +314,7 @@ We use `prisma-json-types-generator` to enforce strict type-safety for Prisma `J
|
||||
|
||||
- Development: Use `bun --bun run prisma migrate dev` to create and apply migrations during development.
|
||||
- Production: Use `bun --bun run prisma migrate deploy` to apply pending migrations in production environments.
|
||||
- **No Manual Migration Creation**: Never create migration SQL files or directories manually by hand. Always use Prisma CLI commands (e.g. `bun --bun run prisma migrate dev --create-only` to generate a migration template, or `bun --bun run prisma migrate dev`) so Prisma correctly tracks migration metadata and checksums.
|
||||
- **No Automatic Dev DB Reset**: Do not run `prisma migrate reset --force` or commands that force-reset the database automatically. If migrations become out of sync or a reset is required, stop executing, report the situation to the user, and present suggested manual cleanup/reset steps for the user to execute.
|
||||
|
||||
### Commands
|
||||
|
||||
@@ -49,6 +49,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
bubblewrap \
|
||||
gosu \
|
||||
poppler-utils \
|
||||
fonts-noto-cjk \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a target="_blank" href="https://shumai.one/">Website</a> | <a target="_blank" href="https://docs.shumai.one/introduction">Docs</a> | <a target="_blank" href="https://staging.shumai.one">Demo</a> | <a target="_blank" href="README_CN.md">中文</a>
|
||||
<a target="_blank" href="https://shumai.one/">Website</a> | <a target="_blank" href="https://docs.shumai.one/introduction">Docs</a> | <a target="_blank" href="https://staging.shumai.one">Demo</a> | <a target="_blank" href="https://discord.gg/sUzhujK6DD">Discord</a> | <a target="_blank" href="README_CN.md">中文</a>
|
||||
</p>
|
||||
|
||||

|
||||
@@ -125,23 +125,24 @@ mkdir shumai && cd shumai
|
||||
Shumai requires the following system packages on Linux:
|
||||
|
||||
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
|
||||
- **`poppler`** (`poppler-utils`) – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
|
||||
- **`bubblewrap`**, **`socat`**, and **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`) for process isolation, networking, and workspace search.
|
||||
|
||||
Install all required packages with one command:
|
||||
|
||||
> Ubuntu/Debian
|
||||
```bash
|
||||
sudo apt install -y ffmpeg bubblewrap socat ripgrep
|
||||
sudo apt install -y ffmpeg poppler-utils bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Fedora
|
||||
```bash
|
||||
sudo dnf install -y ffmpeg bubblewrap socat ripgrep
|
||||
sudo dnf install -y ffmpeg poppler-utils bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Arch Linux
|
||||
```bash
|
||||
sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
|
||||
sudo pacman -S --noconfirm ffmpeg poppler bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
@@ -162,12 +163,13 @@ sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
|
||||
Shumai requires:
|
||||
|
||||
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
|
||||
- **`poppler`** – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
|
||||
- **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`).
|
||||
|
||||
Install the required packages with Homebrew:
|
||||
|
||||
```bash
|
||||
brew install ffmpeg ripgrep
|
||||
brew install ffmpeg poppler ripgrep
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+6
-4
@@ -139,23 +139,24 @@ mkdir shumai && cd shumai
|
||||
在 Linux 上运行 Shumai 需要安装以下系统依赖:
|
||||
|
||||
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
|
||||
- **`poppler`** (`poppler-utils`) —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
|
||||
- **`bubblewrap`**、**`socat`** 和 **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需,用于进程隔离、网络通信和工作区搜索。
|
||||
|
||||
可以使用以下命令一次性安装所有依赖:
|
||||
|
||||
> Ubuntu/Debian
|
||||
```bash
|
||||
sudo apt install -y ffmpeg bubblewrap socat ripgrep
|
||||
sudo apt install -y ffmpeg poppler-utils bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Fedora
|
||||
```bash
|
||||
sudo dnf install -y ffmpeg bubblewrap socat ripgrep
|
||||
sudo dnf install -y ffmpeg poppler-utils bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Arch Linux
|
||||
```bash
|
||||
sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
|
||||
sudo pacman -S --noconfirm ffmpeg poppler bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
@@ -176,12 +177,13 @@ sudo pacman -S --noconfirm ffmpeg bubblewrap socat ripgrep
|
||||
在 macOS 上运行 Shumai 需要:
|
||||
|
||||
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
|
||||
- **`poppler`** —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
|
||||
- **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需。
|
||||
|
||||
使用 Homebrew 安装所需依赖:
|
||||
|
||||
```bash
|
||||
brew install ffmpeg ripgrep
|
||||
brew install ffmpeg poppler ripgrep
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -33,6 +33,8 @@ export function getMediaType(filename: string): string {
|
||||
'.wav': 'audio/wav',
|
||||
'.ogg': 'audio/ogg',
|
||||
'.txt': 'text/plain',
|
||||
'.md': 'text/markdown',
|
||||
'.markdown': 'text/markdown',
|
||||
'.json': 'application/json',
|
||||
'.pdf': 'application/pdf',
|
||||
'.zip': 'application/zip',
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
+23
-1
@@ -1,5 +1,6 @@
|
||||
import { loadEnvConfig } from '@shumai/core/src/env-loader'
|
||||
import { join } from 'node:path'
|
||||
import { readdirSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
|
||||
import index from '@shumai/webui/index.html'
|
||||
@@ -84,6 +85,27 @@ async function run() {
|
||||
routes[routePath] = () => new Response(Bun.file(filePath), { headers: file.headers })
|
||||
}
|
||||
|
||||
const sampleFile = bundle.files[0]?.path || bundle.index || ''
|
||||
const bundleDir = join(import.meta.dir, dirname(sampleFile))
|
||||
try {
|
||||
const dirFiles = readdirSync(bundleDir)
|
||||
const pdfWorkerFile = dirFiles.find((f) => f.startsWith('pdf.worker'))
|
||||
if (pdfWorkerFile) {
|
||||
const routePath = '/' + pdfWorkerFile
|
||||
const filePath = join(bundleDir, pdfWorkerFile)
|
||||
const bunFile = Bun.file(filePath)
|
||||
routes[routePath] = () =>
|
||||
new Response(bunFile, {
|
||||
headers: {
|
||||
'content-type': 'text/javascript;charset=utf-8',
|
||||
etag: `"${bunFile.size}-${bunFile.lastModified}"`,
|
||||
},
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('Failed to register pdf.worker route:', err)
|
||||
}
|
||||
|
||||
const mainHtmlPath = join(import.meta.dir, bundle.index)
|
||||
const mainHtmlFile = Bun.file(mainHtmlPath)
|
||||
const mainHtmlHeaders = bundle.files.find((f) => f.path === bundle.index)?.headers || {
|
||||
|
||||
@@ -34,6 +34,7 @@ await Bun.build({
|
||||
'@temporalio/client',
|
||||
'@temporalio/worker',
|
||||
'@temporalio/workflow',
|
||||
'pdfkit',
|
||||
],
|
||||
// compile: {
|
||||
// outfile: 'Shumai',
|
||||
|
||||
@@ -4,10 +4,6 @@
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "shumai",
|
||||
"dependencies": {
|
||||
"@tanstack/react-virtual": "3.14.2",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "10.0.1",
|
||||
"@inlang/paraglide-js": "2.20.0",
|
||||
@@ -37,7 +33,7 @@
|
||||
},
|
||||
"apps/agent": {
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"bin": {
|
||||
"shumai-agent": "./src/index.ts",
|
||||
},
|
||||
@@ -49,7 +45,7 @@
|
||||
},
|
||||
"apps/cli": {
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"bin": {
|
||||
"shumai-cli": "./src/index.ts",
|
||||
},
|
||||
@@ -64,7 +60,7 @@
|
||||
},
|
||||
"apps/transcode": {
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"bin": {
|
||||
"shumai-transcode": "./src/index.ts",
|
||||
},
|
||||
@@ -76,7 +72,7 @@
|
||||
},
|
||||
"apps/web": {
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/api": "workspace:*",
|
||||
@@ -96,7 +92,7 @@
|
||||
},
|
||||
"packages/agent": {
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.52",
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
@@ -111,6 +107,7 @@
|
||||
"@temporalio/worker": "1.17.2",
|
||||
"@temporalio/workflow": "1.17.2",
|
||||
"adm-zip": "0.5.17",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.4.3",
|
||||
},
|
||||
@@ -120,7 +117,7 @@
|
||||
},
|
||||
"packages/api": {
|
||||
"name": "@shumai/api",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@hono/zod-validator": "0.7.6",
|
||||
"@shumai/core": "workspace:*",
|
||||
@@ -134,7 +131,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@shumai/core",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -142,12 +139,14 @@
|
||||
"@sinclair/typebox": "0.34.49",
|
||||
"adm-zip": "0.5.17",
|
||||
"better-auth": "1.6.11",
|
||||
"csv-parse": "7.0.1",
|
||||
"date-fns": "4.4.0",
|
||||
"dotenv": "16.6.1",
|
||||
"dotenv-expand": "13.0.0",
|
||||
"extract-zip": "2.0.1",
|
||||
"hono": "4.12.23",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"pdfkit": "0.19.1",
|
||||
"pino": "10.3.1",
|
||||
"sharp": "0.34.5",
|
||||
"tweetnacl": "1.0.3",
|
||||
@@ -158,11 +157,12 @@
|
||||
"devDependencies": {
|
||||
"@types/adm-zip": "0.5.8",
|
||||
"@types/extract-zip": "2.0.3",
|
||||
"@types/pdfkit": "0.17.6",
|
||||
},
|
||||
},
|
||||
"packages/db": {
|
||||
"name": "@shumai/db",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@prisma/adapter-pg": "7.8.0",
|
||||
@@ -178,19 +178,33 @@
|
||||
},
|
||||
"packages/dtos": {
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"zod": "4.4.3",
|
||||
},
|
||||
},
|
||||
"packages/e2e": {
|
||||
"name": "@shumai/e2e",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@google/genai": "1.52.0",
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@shumai/transcode": "workspace:*",
|
||||
"@shumai/workflow-core": "workspace:*",
|
||||
},
|
||||
},
|
||||
"packages/tableprinter": {
|
||||
"name": "@shumai/tableprinter",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
},
|
||||
"packages/transcode": {
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -204,7 +218,7 @@
|
||||
},
|
||||
"packages/webui": {
|
||||
"name": "@shumai/webui",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@base-ui/react": "1.5.0",
|
||||
"@dnd-kit/abstract": "0.4.0",
|
||||
@@ -266,6 +280,7 @@
|
||||
"lexical": "0.42.0",
|
||||
"lucide-react": "1.17.0",
|
||||
"next-themes": "0.4.6",
|
||||
"pdfjs-dist": "6.1.200",
|
||||
"radix-ui": "1.4.3",
|
||||
"react": "19.2.0",
|
||||
"react-day-picker": "10.0.1",
|
||||
@@ -301,7 +316,7 @@
|
||||
},
|
||||
"packages/workflow-core": {
|
||||
"name": "@shumai/workflow-core",
|
||||
"version": "0.1.8",
|
||||
"version": "0.1.10",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -718,6 +733,30 @@
|
||||
|
||||
"@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="],
|
||||
|
||||
"@napi-rs/canvas": ["@napi-rs/canvas@1.0.2", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.2", "@napi-rs/canvas-darwin-arm64": "1.0.2", "@napi-rs/canvas-darwin-x64": "1.0.2", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", "@napi-rs/canvas-linux-arm64-musl": "1.0.2", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-musl": "1.0.2", "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", "@napi-rs/canvas-win32-x64-msvc": "1.0.2" } }, "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ=="],
|
||||
|
||||
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.2", "", { "os": "android", "cpu": "arm64" }, "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g=="],
|
||||
|
||||
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ=="],
|
||||
|
||||
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA=="],
|
||||
|
||||
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA=="],
|
||||
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw=="],
|
||||
|
||||
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g=="],
|
||||
|
||||
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q=="],
|
||||
|
||||
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ=="],
|
||||
|
||||
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A=="],
|
||||
|
||||
"@noble/ciphers": ["@noble/ciphers@2.2.0", "", {}, "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA=="],
|
||||
|
||||
"@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="],
|
||||
@@ -1010,6 +1049,8 @@
|
||||
|
||||
"@shumai/dtos": ["@shumai/dtos@workspace:packages/dtos"],
|
||||
|
||||
"@shumai/e2e": ["@shumai/e2e@workspace:packages/e2e"],
|
||||
|
||||
"@shumai/tableprinter": ["@shumai/tableprinter@workspace:packages/tableprinter"],
|
||||
|
||||
"@shumai/transcode": ["@shumai/transcode@workspace:packages/transcode"],
|
||||
@@ -1076,6 +1117,8 @@
|
||||
|
||||
"@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="],
|
||||
|
||||
"@swc/helpers": ["@swc/helpers@0.5.23", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw=="],
|
||||
|
||||
"@swc/types": ["@swc/types@0.1.26", "", { "dependencies": { "@swc/counter": "^0.1.3" } }, "sha512-lyMwd7WGgG79RS7EERZV3T8wMdmPq3xwyg+1nmAM64kIhx5yl+juO2PYIHb7vTiPgPCj8LYjsNV2T5wiQHUEaw=="],
|
||||
|
||||
"@tailwindcss/node": ["@tailwindcss/node@4.3.0", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.21.0", "jiti": "^2.6.1", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.0" } }, "sha512-aFb4gUhFOgdh9AXo4IzBEOzBkkAxm9VigwDJnMIYv3lcfXCJVesNfbEaBl4BNgVRyid92AmdviqwBUBRKSeY3g=="],
|
||||
@@ -1222,6 +1265,8 @@
|
||||
|
||||
"@types/node": ["@types/node@25.9.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg=="],
|
||||
|
||||
"@types/pdfkit": ["@types/pdfkit@0.17.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-tIwzxk2uWKp0Cq9JIluQXJid77lYhF52EsIOwhsMF4iWLA6YneoBR1xVKYYdAysHuepUB0OX4tdwMiUDdGKmig=="],
|
||||
|
||||
"@types/pg": ["@types/pg@8.20.0", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow=="],
|
||||
|
||||
"@types/react": ["@types/react@19.2.16", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-esJiCAnl0kfpNdE69f3So4WJUXy95dLZydX0KwK46riIHDzHM7O9Vtf9xCHW0PXIqvgqNrswl522kA/5yx+F4w=="],
|
||||
@@ -1432,6 +1477,10 @@
|
||||
|
||||
"brace-expansion": ["brace-expansion@5.0.6", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g=="],
|
||||
|
||||
"brotli": ["brotli@1.3.3", "", { "dependencies": { "base64-js": "^1.1.2" } }, "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg=="],
|
||||
|
||||
"browserify-zlib": ["browserify-zlib@0.2.0", "", { "dependencies": { "pako": "~1.0.5" } }, "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA=="],
|
||||
|
||||
"browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="],
|
||||
|
||||
"buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="],
|
||||
@@ -1490,6 +1539,8 @@
|
||||
|
||||
"cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="],
|
||||
|
||||
"clone": ["clone@2.1.2", "", {}, "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w=="],
|
||||
|
||||
"clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
|
||||
|
||||
"cmdk": ["cmdk@1.1.1", "", { "dependencies": { "@radix-ui/react-compose-refs": "^1.1.1", "@radix-ui/react-dialog": "^1.1.6", "@radix-ui/react-id": "^1.1.0", "@radix-ui/react-primitive": "^2.0.2" }, "peerDependencies": { "react": "^18 || ^19 || ^19.0.0-rc", "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, "sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg=="],
|
||||
@@ -1530,6 +1581,8 @@
|
||||
|
||||
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
|
||||
|
||||
"csv-parse": ["csv-parse@7.0.1", "", {}, "sha512-+2z7Ar0APQ7Uu6fX4cn+pitRmxjZ1WPBcGmZFKmA74FCyi7Et/XZx8cjNQ5CjbZ4HCOxXCOpRBYvYH08Qa003A=="],
|
||||
|
||||
"d3-array": ["d3-array@3.2.4", "", { "dependencies": { "internmap": "1 - 2" } }, "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg=="],
|
||||
|
||||
"d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="],
|
||||
@@ -1594,6 +1647,8 @@
|
||||
|
||||
"devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="],
|
||||
|
||||
"dfa": ["dfa@1.2.0", "", {}, "sha512-ED3jP8saaweFTjeGX8HQPjeC1YYyZs98jGNZx6IiBvxW7JG5v492kamAQB3m2wop07CvU/RQmzcKr6bgcC5D/Q=="],
|
||||
|
||||
"diff": ["diff@8.0.4", "", {}, "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw=="],
|
||||
|
||||
"docker-compose": ["docker-compose@1.4.2", "", { "dependencies": { "yaml": "^2.2.2" } }, "sha512-rPHigTKGaEHpkUmfd69QgaOp+Os5vGJwG/Ry8lcr8W/382AmI+z/D7qoa9BybKIkqNppaIbs8RYeHSevdQjWww=="],
|
||||
@@ -1746,6 +1801,8 @@
|
||||
|
||||
"flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="],
|
||||
|
||||
"fontkit": ["fontkit@2.0.4", "", { "dependencies": { "@swc/helpers": "^0.5.12", "brotli": "^1.3.2", "clone": "^2.1.2", "dfa": "^1.2.0", "fast-deep-equal": "^3.1.3", "restructure": "^3.0.0", "tiny-inflate": "^1.0.3", "unicode-properties": "^1.4.0", "unicode-trie": "^2.0.0" } }, "sha512-syetQadaUEDNdxdugga9CpEYVaQIxOwk7GlwZWWZ19//qW4zE5bknOKeMBDYAASwnpaSHKJITRLMF9m1fp3s6g=="],
|
||||
|
||||
"for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="],
|
||||
|
||||
"foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="],
|
||||
@@ -1972,6 +2029,8 @@
|
||||
|
||||
"joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="],
|
||||
|
||||
"js-md5": ["js-md5@0.8.3", "", {}, "sha512-qR0HB5uP6wCuRMrWPTrkMaev7MJZwJuuw4fnwAzRgP4J4/F8RwtodOKpGp4XpqsLBFzzgqIO42efFAyz2Et6KQ=="],
|
||||
|
||||
"js-sha256": ["js-sha256@0.11.1", "", {}, "sha512-o6WSo/LUvY2uC4j7mO50a2ms7E/EAdbP0swigLV+nzHKTTaYnaLIWJ02VdXrsJX0vGedDESQnLsOekr94ryfjg=="],
|
||||
|
||||
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
@@ -2034,6 +2093,8 @@
|
||||
|
||||
"lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="],
|
||||
|
||||
"linebreak": ["linebreak@1.1.0", "", { "dependencies": { "base64-js": "0.0.8", "unicode-trie": "^2.0.0" } }, "sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ=="],
|
||||
|
||||
"loader-runner": ["loader-runner@4.3.2", "", {}, "sha512-DFEqQ3ihfS9blba08cLfYf1NRAIEm+dDjic073DRDc3/JspI/8wYmtDsHwd3+4hwvdxSK7PGaElfTmm0awWJ4w=="],
|
||||
|
||||
"locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="],
|
||||
@@ -2210,6 +2271,8 @@
|
||||
|
||||
"package-json-from-dist": ["package-json-from-dist@1.0.1", "", {}, "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="],
|
||||
|
||||
"pako": ["pako@0.2.9", "", {}, "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA=="],
|
||||
|
||||
"parse-entities": ["parse-entities@4.0.2", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="],
|
||||
|
||||
"partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="],
|
||||
@@ -2226,6 +2289,10 @@
|
||||
|
||||
"pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
|
||||
|
||||
"pdfjs-dist": ["pdfjs-dist@6.1.200", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.0" } }, "sha512-o8MolyzirkkLrcdsae/HEOiIcXWI7DS5zGpvqW8xTC2YUsW30rltFw2bDGvw/fskUdEMrQm2br68jzDS5BH2vw=="],
|
||||
|
||||
"pdfkit": ["pdfkit@0.19.1", "", { "dependencies": { "@noble/ciphers": "^1.0.0", "@noble/hashes": "^1.6.0", "fontkit": "^2.0.4", "js-md5": "^0.8.3", "linebreak": "^1.1.0", "png-js": "^1.1.0" } }, "sha512-6Gzk+wDwTs4VSxsR5rCMTnIl5nlmkye1oWB0l2hDB1EX6ZNSIBroKQEv+2+fPPn+stVjyqzmsqRJVDfB9fo5DA=="],
|
||||
|
||||
"pend": ["pend@1.2.0", "", {}, "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="],
|
||||
|
||||
"perfect-debounce": ["perfect-debounce@2.1.0", "", {}, "sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g=="],
|
||||
@@ -2266,6 +2333,8 @@
|
||||
|
||||
"playwright-core": ["playwright-core@1.61.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg=="],
|
||||
|
||||
"png-js": ["png-js@1.1.0", "", { "dependencies": { "browserify-zlib": "^0.2.0" } }, "sha512-PM/uYGzGdNSzqeOgly68+6wKQDL1SY0a/N+OEa/+br6LnHWOAJB0Npiamnodfq3jd2LS/i2fMeOKSAILjA+m5Q=="],
|
||||
|
||||
"possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="],
|
||||
|
||||
"postcss": ["postcss@8.5.15", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A=="],
|
||||
@@ -2388,6 +2457,8 @@
|
||||
|
||||
"resolve": ["resolve@2.0.0-next.7", "", { "dependencies": { "es-errors": "^1.3.0", "is-core-module": "^2.16.2", "node-exports-info": "^1.6.0", "object-keys": "^1.1.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-tqt+NBWwyaMgw3zDsnygx4CByWjQEJHOPMdslYhppaQSJUtL/D4JO9CcBBlhPoI8lz9oJIDXkwXfhF4aWqP8xQ=="],
|
||||
|
||||
"restructure": ["restructure@3.0.2", "", {}, "sha512-gSfoiOEA0VPE6Tukkrr7I0RBdE0s7H1eFCDBk05l1KIQT1UIKNc5JZy6jdyW6eYH3aR3g5b3PuL77rq0hvwtAw=="],
|
||||
|
||||
"retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="],
|
||||
|
||||
"rollup": ["rollup@4.61.0", "", { "dependencies": { "@types/estree": "1.0.9" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.61.0", "@rollup/rollup-android-arm64": "4.61.0", "@rollup/rollup-darwin-arm64": "4.61.0", "@rollup/rollup-darwin-x64": "4.61.0", "@rollup/rollup-freebsd-arm64": "4.61.0", "@rollup/rollup-freebsd-x64": "4.61.0", "@rollup/rollup-linux-arm-gnueabihf": "4.61.0", "@rollup/rollup-linux-arm-musleabihf": "4.61.0", "@rollup/rollup-linux-arm64-gnu": "4.61.0", "@rollup/rollup-linux-arm64-musl": "4.61.0", "@rollup/rollup-linux-loong64-gnu": "4.61.0", "@rollup/rollup-linux-loong64-musl": "4.61.0", "@rollup/rollup-linux-ppc64-gnu": "4.61.0", "@rollup/rollup-linux-ppc64-musl": "4.61.0", "@rollup/rollup-linux-riscv64-gnu": "4.61.0", "@rollup/rollup-linux-riscv64-musl": "4.61.0", "@rollup/rollup-linux-s390x-gnu": "4.61.0", "@rollup/rollup-linux-x64-gnu": "4.61.0", "@rollup/rollup-linux-x64-musl": "4.61.0", "@rollup/rollup-openbsd-x64": "4.61.0", "@rollup/rollup-openharmony-arm64": "4.61.0", "@rollup/rollup-win32-arm64-msvc": "4.61.0", "@rollup/rollup-win32-ia32-msvc": "4.61.0", "@rollup/rollup-win32-x64-gnu": "4.61.0", "@rollup/rollup-win32-x64-msvc": "4.61.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-T9mWdbWfQtp0B5lv/HX+wrhYsmXRlcWnXXmJbXqKJhlRaoS6KMhq0gpyzW4UJfclcxrEdLnTgjT2NjruLONu0g=="],
|
||||
@@ -2550,6 +2621,8 @@
|
||||
|
||||
"thread-stream": ["thread-stream@4.2.0", "", { "dependencies": { "real-require": "^1.0.0" } }, "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ=="],
|
||||
|
||||
"tiny-inflate": ["tiny-inflate@1.0.3", "", {}, "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw=="],
|
||||
|
||||
"tiny-invariant": ["tiny-invariant@1.3.3", "", {}, "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="],
|
||||
|
||||
"tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="],
|
||||
@@ -2606,6 +2679,10 @@
|
||||
|
||||
"undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],
|
||||
|
||||
"unicode-properties": ["unicode-properties@1.4.1", "", { "dependencies": { "base64-js": "^1.3.0", "unicode-trie": "^2.0.0" } }, "sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg=="],
|
||||
|
||||
"unicode-trie": ["unicode-trie@2.0.0", "", { "dependencies": { "pako": "^0.2.5", "tiny-inflate": "^1.0.0" } }, "sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ=="],
|
||||
|
||||
"unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="],
|
||||
|
||||
"unionfs": ["unionfs@4.6.0", "", { "dependencies": { "fs-monkey": "^1.0.0" } }, "sha512-fJAy3gTHjFi5S3TP5EGdjs/OUMFFvI/ady3T8qVuZfkv8Qi8prV/Q8BuFEgODJslhZTT2z2qdD2lGdee9qjEnA=="],
|
||||
@@ -3010,6 +3087,8 @@
|
||||
|
||||
"bl/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="],
|
||||
|
||||
"browserify-zlib/pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
|
||||
|
||||
"bun-types/@types/node": ["@types/node@22.19.19", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew=="],
|
||||
|
||||
"c12/dotenv": ["dotenv@17.4.2", "", {}, "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw=="],
|
||||
@@ -3038,12 +3117,18 @@
|
||||
|
||||
"lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"linebreak/base64-js": ["base64-js@0.0.8", "", {}, "sha512-3XSA2cR/h/73EzlXXdU6YNycmYI7+kicTxks4eJg2g39biHR84slg2+des+p7iHYhbRg/udIS4TD53WabcOUkw=="],
|
||||
|
||||
"p-retry/retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="],
|
||||
|
||||
"parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="],
|
||||
|
||||
"path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
||||
|
||||
"pdfkit/@noble/ciphers": ["@noble/ciphers@1.3.0", "", {}, "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw=="],
|
||||
|
||||
"pdfkit/@noble/hashes": ["@noble/hashes@1.8.0", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="],
|
||||
|
||||
"pg-types/postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="],
|
||||
|
||||
"pretty-format/react-is": ["react-is@17.0.2", "", {}, "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w=="],
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
"using-shumai/agent-providers",
|
||||
"using-shumai/agents",
|
||||
"using-shumai/agent-skills",
|
||||
"using-shumai/agent-tools",
|
||||
"using-shumai/agent-sandbox"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
---
|
||||
title: Agent Tools
|
||||
description: Configure the tools and capabilities permitted for each agent.
|
||||
---
|
||||
|
||||
Shumai agents are equipped with built-in tools that allow them to interact with the file system, view files, extract video frames, and execute commands.
|
||||
|
||||
By default, **all tools are enabled** for new agents. Team Owners can deny specific tools to restrict agent capabilities.
|
||||
|
||||
<Warning>
|
||||
**Caution:** Disabling tools can cause agents or skills to fail. For example, disabling the
|
||||
**Sandboxed Terminal** tool will break almost all custom skills that rely on executing commands or
|
||||
script environments. Disable tools only when you have a specific security policy or isolation
|
||||
requirement.
|
||||
</Warning>
|
||||
|
||||
---
|
||||
|
||||
## Available Tools
|
||||
|
||||
The following tools can be individually toggled in the agent configuration dialog:
|
||||
|
||||
### 1. Sandboxed Terminal (`bash`)
|
||||
|
||||
- **Description:** Allows the agent to run terminal/shell commands in a highly isolated environment.
|
||||
- **Why it matters:** Essential for running custom scripts, processing files on disk, or using standard Unix utilities. If disabled, the agent won't be able to run command-line code.
|
||||
|
||||
### 2. Read Skills (`read_skill`)
|
||||
|
||||
- **Description:** Allows the agent to load and read skill definitions to perform specialized actions.
|
||||
- **Why it matters:** Enables the agent to inspect the instructions/configurations of custom skills registered in your workspace before executing them.
|
||||
|
||||
### 3. Analyze Image (`analyze_image`)
|
||||
|
||||
- **Description:** Allows the agent to view and analyze the visual contents of an image file.
|
||||
- **Why it matters:** Enables multimodal agents to "see" image assets uploaded to Shumai, allowing them to answer comments or write metadata about images.
|
||||
|
||||
### 4. Video Frame Extraction (`screenshot`)
|
||||
|
||||
- **Description:** Allows the agent to extract still image frames from video files for analysis.
|
||||
- **Why it matters:** Enables the agent to perform frame-by-frame analysis on videos (e.g. for comments, annotation, or search).
|
||||
|
||||
### 5. Browse Shumai (`list_assets`)
|
||||
|
||||
- **Description:** Allows the agent to list folders and browse files in Shumai.
|
||||
- **Why it matters:** Gives the agent visual mapping of the project structure so it can search for other files or locate related assets.
|
||||
|
||||
### 6. Create Folder (`create_folder`)
|
||||
|
||||
- **Description:** Allows the agent to create new folders in Shumai.
|
||||
- **Why it matters:** Needed if the agent organizes outputs or places files in custom subdirectory folders.
|
||||
|
||||
### 7. Create File (`create_file`)
|
||||
|
||||
- **Description:** Allows the agent to create and upload new files to Shumai.
|
||||
- **Why it matters:** Allows the agent to generate new assets (e.g. text logs, transcoded clips, reports) and save them directly back into your project.
|
||||
|
||||
### 8. Create Version (`create_version`)
|
||||
|
||||
- **Description:** Allows the agent to upload and stack a new version of an existing file in Shumai.
|
||||
- **Why it matters:** Allows the agent to update files in a clean, non-destructive way by stacking revisions rather than creating duplicate files.
|
||||
|
||||
---
|
||||
|
||||
## Restricting Tools
|
||||
|
||||
To change which tools an agent can use:
|
||||
|
||||
1. Navigate to **Team Settings** > **Agents** (Note: Only Team Owners can view and modify these settings).
|
||||
2. Click on the agent you want to edit.
|
||||
3. Scroll down below the **Skills** section to find the **Tools** configuration panel.
|
||||
4. Toggle the switches next to the tools to enable or disable them.
|
||||
5. Click **Save Changes**.
|
||||
@@ -76,6 +76,7 @@ export default tseslint.config(
|
||||
'**/data/**',
|
||||
'**/packages/webui/components/ui/**',
|
||||
'**/packages/webui/paraglide/**',
|
||||
'**/packages/webui/public/**',
|
||||
],
|
||||
},
|
||||
js.configs.recommended,
|
||||
|
||||
+2
-5
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
@@ -19,6 +19,7 @@
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
||||
"test": "bun --bun vitest run",
|
||||
"test:e2e": "playwright test",
|
||||
"test:e2e:workflow": "bun --bun vitest run -c packages/e2e/vitest.config.ts",
|
||||
"test:e2e:ui": "playwright test --ui",
|
||||
"test:e2e:report": "playwright show-report",
|
||||
"test:e2e:install": "playwright install --with-deps",
|
||||
@@ -65,9 +66,5 @@
|
||||
"typescript-eslint": "8.60.1",
|
||||
"vite-tsconfig-paths": "6.1.1",
|
||||
"vitest": "4.1.8"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tanstack/react-virtual": "3.14.2",
|
||||
"jittered-fractional-indexing": "1.0.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
@@ -20,7 +20,8 @@
|
||||
"@sinclair/typebox": "0.34.49",
|
||||
"ulid": "3.0.2",
|
||||
"adm-zip": "0.5.17",
|
||||
"zod": "4.4.3"
|
||||
"zod": "4.4.3",
|
||||
"jittered-fractional-indexing": "1.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/adm-zip": "0.5.8"
|
||||
|
||||
@@ -596,6 +596,45 @@ describe('Agent Database Activities Integration', () => {
|
||||
const fetchedComment = await getCommentActivity(comment.id)
|
||||
expect(fetchedComment?.id).toBe(comment.id)
|
||||
})
|
||||
|
||||
it('should resolve version stack asset to its latest version file', async () => {
|
||||
const stack = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'Stack',
|
||||
type: AssetType.version_stack,
|
||||
status: AssetStatus.uploaded,
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'v1.png',
|
||||
type: AssetType.file,
|
||||
mediaType: 'image/png',
|
||||
status: AssetStatus.uploaded,
|
||||
projectId: project.id,
|
||||
parentId: stack.id,
|
||||
sortIndex: 'a1',
|
||||
},
|
||||
})
|
||||
|
||||
const v2 = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'v2.png',
|
||||
type: AssetType.file,
|
||||
mediaType: 'image/png',
|
||||
status: AssetStatus.uploaded,
|
||||
projectId: project.id,
|
||||
parentId: stack.id,
|
||||
sortIndex: 'a0',
|
||||
},
|
||||
})
|
||||
|
||||
const fetchedAsset = await getAssetActivity(stack.id)
|
||||
expect(fetchedAsset?.id).toBe(v2.id)
|
||||
expect(fetchedAsset?.name).toBe('v2.png')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getAssetPathContextActivity', () => {
|
||||
|
||||
@@ -763,7 +763,7 @@ export async function getAgentAutofillContextActivity(params: {
|
||||
}
|
||||
|
||||
export async function getAssetActivity(assetId: string) {
|
||||
return prisma.asset.findUnique({
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: {
|
||||
storageKey: true,
|
||||
@@ -774,6 +774,30 @@ export async function getAssetActivity(assetId: string) {
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if (asset && asset.type === AssetType.version_stack) {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: {
|
||||
storageKey: true,
|
||||
project: {
|
||||
include: {
|
||||
team: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if (latestVersion) {
|
||||
return {
|
||||
...latestVersion,
|
||||
project: latestVersion.project ?? asset.project,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return asset
|
||||
}
|
||||
|
||||
export async function getCommentActivity(commentId: string) {
|
||||
|
||||
@@ -88,8 +88,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const isImage = asset.mediaType.startsWith('image/')
|
||||
const isVideo = asset.mediaType.startsWith('video/')
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
throw ApplicationFailure.create({
|
||||
|
||||
@@ -409,6 +409,8 @@ describe('DatabaseSessionStorage', () => {
|
||||
name: 'test.png',
|
||||
type: 'file',
|
||||
mediaType: 'image/png',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
|
||||
media: { proxyType: 'image' } as any,
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
@@ -443,9 +445,9 @@ describe('DatabaseSessionStorage', () => {
|
||||
providers: [],
|
||||
})
|
||||
|
||||
// 4. Assert that the tools were instantiated using the new comment ID, not the original one
|
||||
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(asset.id, 'new-comment-456')
|
||||
expect(createScreenshotToolSpy).not.toHaveBeenCalled()
|
||||
// 4. Assert that the tools were instantiated using the new comment ID and user ID
|
||||
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
})
|
||||
|
||||
it('should use the current comment ID for media tools in subsequent turns (video)', async () => {
|
||||
@@ -461,6 +463,8 @@ describe('DatabaseSessionStorage', () => {
|
||||
name: 'test.mp4',
|
||||
type: 'file',
|
||||
mediaType: 'video/mp4',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
|
||||
media: { proxyType: 'video' } as any,
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
@@ -495,8 +499,8 @@ describe('DatabaseSessionStorage', () => {
|
||||
providers: [],
|
||||
})
|
||||
|
||||
// 4. Assert that the tools were instantiated using the new comment ID, not the original one
|
||||
expect(createScreenshotToolSpy).toHaveBeenCalledWith(asset.id, 'new-comment-456')
|
||||
expect(createAnalyzeImageToolSpy).not.toHaveBeenCalled()
|
||||
// 4. Assert that the tools were instantiated using the new comment ID and user ID
|
||||
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -43,6 +43,7 @@ describe('formatSkillsForPrompt', () => {
|
||||
// Verify lines and XML elements
|
||||
expect(result).toContain('<available_skills>')
|
||||
expect(result).toContain('<skill>')
|
||||
expect(result).toContain('<id>skill-1</id>')
|
||||
expect(result).toContain('<name>Deploy & Test</name>')
|
||||
expect(result).toContain(
|
||||
'<description>Deploys code <deploy> and "runs" tests.</description>',
|
||||
@@ -329,4 +330,47 @@ describe('Sandbox Network isolation integration', () => {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('should filter out denied tools from AgentHarness', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team 3' },
|
||||
})
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'test-agent-3',
|
||||
name: 'Agent 3',
|
||||
email: 'agent3@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'test-agent-3',
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'google', model: 'gemini', deniedTools: ['bash'] },
|
||||
},
|
||||
})
|
||||
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
try {
|
||||
const { harness } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
agentId: 'test-agent-3',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: [],
|
||||
})
|
||||
const tools = harness.getTools()
|
||||
const bashTool = tools.find((t) => t.name === 'bash')
|
||||
expect(bashTool).toBeUndefined()
|
||||
const readSkillTool = tools.find((t) => t.name === 'read_skill')
|
||||
expect(readSkillTool).toBeDefined()
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
+18
-15
@@ -15,7 +15,7 @@ import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { createAnalyzeImageTool } from './tools/analyze-image'
|
||||
import { createScreenshotTool } from './tools/screenshot'
|
||||
import { getSimpleMediaType } from './workflows/agent-chat-prompt-builder'
|
||||
import { createReadPdfPagesTool } from './tools/read-pdf-pages'
|
||||
import { createCreateFileTool } from './tools/create-file'
|
||||
import { createCreateFolderTool } from './tools/create-folder'
|
||||
import { createCreateVersionTool } from './tools/create-version'
|
||||
@@ -225,23 +225,16 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
}
|
||||
|
||||
const metadata = await storage.getMetadata()
|
||||
const assetId = metadata.assetId
|
||||
const userCommentId =
|
||||
passedUserCommentId !== undefined ? passedUserCommentId : metadata.userCommentId
|
||||
|
||||
const mediaTools: AgentTool[] = []
|
||||
if (assetId) {
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
})
|
||||
if (asset?.mediaType) {
|
||||
const type = getSimpleMediaType(asset.mediaType)
|
||||
if (type === 'image') {
|
||||
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
|
||||
} else if (type === 'video') {
|
||||
mediaTools.push(createScreenshotTool(assetId, userCommentId))
|
||||
}
|
||||
}
|
||||
if (userId) {
|
||||
mediaTools.push(
|
||||
createAnalyzeImageTool(userId, userCommentId),
|
||||
createScreenshotTool(userId, userCommentId),
|
||||
createReadPdfPagesTool(userId, userCommentId),
|
||||
)
|
||||
}
|
||||
|
||||
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs, {
|
||||
@@ -262,6 +255,15 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
)
|
||||
}
|
||||
|
||||
const agent = await prisma.agent.findUnique({
|
||||
where: { id: agentId },
|
||||
})
|
||||
const agentConfig = agent?.config as PrismaJson.AgentConfig | null | undefined
|
||||
const deniedTools = agentConfig?.deniedTools || []
|
||||
|
||||
const allTools = [...mediaTools, readSkill, sandboxedBash, ...systemTools, ...customTools]
|
||||
const enabledTools = allTools.filter((tool) => !deniedTools.includes(tool.name))
|
||||
|
||||
const harness = new AgentHarness({
|
||||
env: new NodeExecutionEnv({ cwd: process.cwd() }),
|
||||
session,
|
||||
@@ -289,7 +291,7 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
getApiKeyAndHeaders: async (targetModel) => {
|
||||
return getApiKeyAndHeadersFromDb(params.providers, targetModel.provider)
|
||||
},
|
||||
tools: [...mediaTools, readSkill, sandboxedBash, ...systemTools, ...customTools],
|
||||
tools: enabledTools,
|
||||
})
|
||||
|
||||
return { session, harness }
|
||||
@@ -362,6 +364,7 @@ export function formatSkillsForPrompt(
|
||||
for (const skill of skills) {
|
||||
const filePath = path.join(process.cwd(), '.pi', 'skills', skill.id, 'SKILL.md')
|
||||
lines.push(' <skill>')
|
||||
lines.push(` <id>${escapeXml(skill.id)}</id>`)
|
||||
lines.push(` <name>${escapeXml(skill.name)}</name>`)
|
||||
lines.push(` <description>${escapeXml(skill.description || 'No description')}</description>`)
|
||||
lines.push(` <location>${escapeXml(filePath)}</location>`)
|
||||
|
||||
@@ -5,16 +5,21 @@ import {
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type AssetComment,
|
||||
type User,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('analyzeImageTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should fetch the image from S3 when comment has no annotations', async () => {
|
||||
it('should check user permissions and fetch the image from S3 when comment has no annotations', async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
@@ -55,6 +71,7 @@ describe('analyzeImageTool', () => {
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
annotation: null,
|
||||
} as unknown as AssetComment)
|
||||
|
||||
@@ -63,8 +80,15 @@ describe('analyzeImageTool', () => {
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createAnalyzeImageTool('asset-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', {})
|
||||
const tool = createAnalyzeImageTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1' })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(result.content[0].type).toBe('image')
|
||||
expect((result.content[0] as unknown as { data: string }).data).toBe(
|
||||
@@ -87,6 +111,7 @@ describe('analyzeImageTool', () => {
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
annotation: [
|
||||
{
|
||||
type: 'box',
|
||||
@@ -114,14 +139,14 @@ describe('analyzeImageTool', () => {
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createAnalyzeImageTool('asset-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', {})
|
||||
const tool = createAnalyzeImageTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1' })
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
type: 'transcode',
|
||||
type: 'transcode_image_annotation',
|
||||
status: 'pending',
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
import { Type } from '@sinclair/typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
const analyzeImageSchema = Type.Object({})
|
||||
const analyzeImageSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'The asset ID of the image to analyze. This parameter is required.',
|
||||
}),
|
||||
})
|
||||
|
||||
export function createAnalyzeImageTool(
|
||||
assetId: string,
|
||||
userId: string,
|
||||
userCommentId?: string | null,
|
||||
): AgentTool<typeof analyzeImageSchema> {
|
||||
return {
|
||||
@@ -17,8 +22,24 @@ export function createAnalyzeImageTool(
|
||||
description:
|
||||
'Retrieves and views the image content. Call this tool if you need to analyze the image.',
|
||||
parameters: analyzeImageSchema,
|
||||
execute: async () => {
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const assetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
@@ -48,13 +69,13 @@ export function createAnalyzeImageTool(
|
||||
}
|
||||
}
|
||||
|
||||
// Check if trigger comment has draw annotations
|
||||
// Check if trigger comment has draw annotations for this asset
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment?.annotation) {
|
||||
if (comment && comment.assetId === assetId && comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
@@ -70,7 +91,7 @@ export function createAnalyzeImageTool(
|
||||
data: {
|
||||
assetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode,
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
|
||||
@@ -34,6 +34,9 @@ function getMimeType(filePath: string): string {
|
||||
'.mp4': 'video/mp4',
|
||||
'.mov': 'video/quicktime',
|
||||
'.pdf': 'application/pdf',
|
||||
'.txt': 'text/plain',
|
||||
'.md': 'text/markdown',
|
||||
'.markdown': 'text/markdown',
|
||||
}
|
||||
return mimeTypes[ext] || 'application/octet-stream'
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@ function getMimeType(filePath: string): string {
|
||||
'.mp4': 'video/mp4',
|
||||
'.mov': 'video/quicktime',
|
||||
'.pdf': 'application/pdf',
|
||||
'.txt': 'text/plain',
|
||||
'.md': 'text/markdown',
|
||||
'.markdown': 'text/markdown',
|
||||
}
|
||||
return mimeTypes[ext] || 'application/octet-stream'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createReadPdfPagesTool } from './read-pdf-pages'
|
||||
import {
|
||||
prisma,
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type AssetComment,
|
||||
type User,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
assetComment: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
workflowTask: {
|
||||
create: vi.fn(),
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
getObject: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/workflow-core', () => ({
|
||||
workflowService: {
|
||||
executeWait: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('readPdfPagesTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
})
|
||||
|
||||
it('should trigger pdfPages transcode workflow and return image outputs', async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
|
||||
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
second: 2,
|
||||
annotation: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
} as unknown as AssetComment)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
pages: [
|
||||
{ key: 'pdf_pages/page_1.webp', page: 1 },
|
||||
{ key: 'pdf_pages/page_2.webp', page: 2 },
|
||||
],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockImplementation(async (_bucket: string, key: string) => {
|
||||
return {
|
||||
buffer: Buffer.from(`fake-bytes-for-${key}`),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string }
|
||||
})
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 2 })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
type: 'transcode_pdf_pages',
|
||||
status: 'pending',
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
pdfPages: {
|
||||
start: 1,
|
||||
end: 2,
|
||||
commentTimestamp: 2,
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.content.length).toBe(2)
|
||||
expect(result.content[0].type).toBe('image')
|
||||
expect((result.content[0] as unknown as { data: string }).data).toBe(
|
||||
Buffer.from('fake-bytes-for-pdf_pages/page_1.webp').toString('base64'),
|
||||
)
|
||||
expect(result.details.sourceKeys).toEqual(['pdf_pages/page_1.webp', 'pdf_pages/page_2.webp'])
|
||||
})
|
||||
|
||||
it('should return error text if start page is less than 1', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Invalid page range: start page (0) must be at least 1.',
|
||||
})
|
||||
})
|
||||
|
||||
it('should return error text if start page is greater than end page', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Invalid page range: start page (5) must be less than or equal to end page (2).',
|
||||
})
|
||||
})
|
||||
|
||||
it('should return error text if page range exceeds maximum limit of 20', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
|
||||
})
|
||||
})
|
||||
|
||||
it('should return error text if asset not found', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Asset with ID asset-1 not found.',
|
||||
})
|
||||
})
|
||||
|
||||
it('should return error text if proxyType is not pdf', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'image' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Asset asset-1 is not a PDF or document.',
|
||||
})
|
||||
})
|
||||
|
||||
it('should return error text if proxyType is null/undefined (unprocessed)', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
media: null,
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Asset asset-1 is not a PDF or document.',
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,186 @@
|
||||
import { Type } from '@sinclair/typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
const readPdfPagesSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'Asset ID of the PDF document. This parameter is required.',
|
||||
}),
|
||||
start: Type.Number({ description: 'Start page number (1-based index)' }),
|
||||
end: Type.Number({ description: 'End page number (1-based index)' }),
|
||||
})
|
||||
|
||||
export function createReadPdfPagesTool(
|
||||
userId: string,
|
||||
userCommentId?: string | null,
|
||||
): AgentTool<typeof readPdfPagesSchema> {
|
||||
return {
|
||||
name: 'read_pdf_pages',
|
||||
label: 'Read PDF Pages',
|
||||
description:
|
||||
'Converts specific page range of a PDF document into 1080p WebP images for visual analysis. Maximum 20 pages allowed per call.',
|
||||
parameters: readPdfPagesSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
if (params.start < 1) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Invalid page range: start page (${params.start}) must be at least 1.`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
if (params.start > params.end) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_PAGE_RANGE = 20
|
||||
const requestedPages = params.end - params.start + 1
|
||||
if (requestedPages > MAX_PAGE_RANGE) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const targetAssetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: targetAssetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: targetAssetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset with ID ${targetAssetId} not found.` }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (proxyType !== 'pdf') {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for comment page and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === targetAssetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger transcode workflow to render PDF pages
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: targetAssetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_pdf_pages,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
pdfPages: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
|
||||
|
||||
if (!pages || pages.length === 0) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'No PDF page images were generated.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const pageItem of pages) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
})
|
||||
sourceKeys.push(pageItem.key)
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error reading PDF pages: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -5,16 +5,21 @@ import {
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type AssetComment,
|
||||
type User,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('screenshotTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should trigger screenshot transcode workflow and return image outputs', async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
@@ -53,6 +69,7 @@ describe('screenshotTool', () => {
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
second: 5.0,
|
||||
annotation: [
|
||||
{
|
||||
@@ -88,14 +105,21 @@ describe('screenshotTool', () => {
|
||||
} as unknown as { buffer: Buffer; contentType: string }
|
||||
})
|
||||
|
||||
const tool = createScreenshotTool('asset-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { start: 0, end: 10, count: 2 })
|
||||
const tool = createScreenshotTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 10, count: 2 })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
type: 'transcode',
|
||||
type: 'transcode_screenshot',
|
||||
status: 'pending',
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
import { Type } from '@sinclair/typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
const screenshotSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'The asset ID of the video asset. This parameter is required.',
|
||||
}),
|
||||
start: Type.Number({ description: 'Start time in seconds' }),
|
||||
end: Type.Number({ description: 'End time in seconds' }),
|
||||
count: Type.Number({ description: 'Number of screenshots to take' }),
|
||||
})
|
||||
|
||||
export function createScreenshotTool(
|
||||
assetId: string,
|
||||
userId: string,
|
||||
userCommentId?: string | null,
|
||||
): AgentTool<typeof screenshotSchema> {
|
||||
return {
|
||||
@@ -23,6 +27,22 @@ export function createScreenshotTool(
|
||||
parameters: screenshotSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const assetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
})
|
||||
@@ -41,7 +61,7 @@ export function createScreenshotTool(
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment) {
|
||||
if (comment && comment.assetId === assetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
@@ -57,7 +77,7 @@ export function createScreenshotTool(
|
||||
data: {
|
||||
assetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode,
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
|
||||
@@ -64,6 +64,7 @@ describe('Agent Autofill Workflow', () => {
|
||||
storageKey: { key: 'asset-key' },
|
||||
project: { id: 'p1', teamId: 't1' },
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
})
|
||||
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
|
||||
filePath: '/tmp/test.png',
|
||||
@@ -290,6 +291,7 @@ describe('Agent Autofill Workflow', () => {
|
||||
storageKey: { key: 'asset-key' },
|
||||
project: { id: 'p1', teamId: 't1' },
|
||||
mediaType: 'audio/mp3',
|
||||
media: { proxyType: 'audio' },
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
|
||||
@@ -64,8 +64,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
const projectId = asset.project.id
|
||||
|
||||
// 2. Prepare Data (Images)
|
||||
const isImage = asset.mediaType?.startsWith('image/') || false
|
||||
const isVideo = asset.mediaType?.startsWith('video/') || false
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
throw ApplicationFailure.create({
|
||||
|
||||
@@ -25,7 +25,13 @@ describe('AgentChatPromptBuilder', () => {
|
||||
})
|
||||
|
||||
it('should build prompt with image asset details', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1').withAssetDetails('test.png', 'image/png')
|
||||
const builder = new AgentChatPromptBuilder('a1').withAssetDetails(
|
||||
'test.png',
|
||||
'image/png',
|
||||
undefined,
|
||||
undefined,
|
||||
'image',
|
||||
)
|
||||
const result = builder.build()
|
||||
|
||||
expect(result).toContain('File Name: test.png')
|
||||
@@ -39,6 +45,8 @@ describe('AgentChatPromptBuilder', () => {
|
||||
'movie.mp4',
|
||||
'video/mp4',
|
||||
12.34,
|
||||
undefined,
|
||||
'video',
|
||||
)
|
||||
const result = builder.build()
|
||||
|
||||
@@ -49,6 +57,45 @@ describe('AgentChatPromptBuilder', () => {
|
||||
expect(result).not.toContain("call the 'analyze_image' tool")
|
||||
})
|
||||
|
||||
it('should build prompt with PDF asset details and total pages', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1').withAssetDetails(
|
||||
'doc.pdf',
|
||||
'application/pdf',
|
||||
undefined,
|
||||
15,
|
||||
'pdf',
|
||||
)
|
||||
const result = builder.build()
|
||||
|
||||
expect(result).toContain('File Name: doc.pdf')
|
||||
expect(result).toContain('File Type: pdf')
|
||||
expect(result).toContain('Total Pages: 15')
|
||||
expect(result).toContain("call the 'read_pdf_pages' tool")
|
||||
expect(result).toContain('Maximum 20 pages allowed per call.')
|
||||
expect(result).not.toContain("call the 'screenshot' tool")
|
||||
expect(result).not.toContain("call the 'analyze_image' tool")
|
||||
})
|
||||
|
||||
it('should build prompt with comment page for PDF asset', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1')
|
||||
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
|
||||
.withCommentTimestamp(3)
|
||||
const result = builder.build()
|
||||
|
||||
expect(result).toContain('Comment Page: 3')
|
||||
})
|
||||
|
||||
it('should build prompt for PDF continuation mode with comment page', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1')
|
||||
.withContinuation(true)
|
||||
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
|
||||
.withCommentTimestamp(4)
|
||||
const result = builder.build()
|
||||
|
||||
expect(result).toContain('Comment Page: 4')
|
||||
expect(result).not.toContain('Comment Timestamp:')
|
||||
})
|
||||
|
||||
it('should build prompt with comment timestamp', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1').withCommentTimestamp(5.5)
|
||||
const result = builder.build()
|
||||
@@ -60,7 +107,7 @@ describe('AgentChatPromptBuilder', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1')
|
||||
.withExplicitMention(true)
|
||||
.withPathContext('src/main.ts')
|
||||
.withAssetDetails('movie.mp4', 'video/mp4', 12.34)
|
||||
.withAssetDetails('movie.mp4', 'video/mp4', 12.34, undefined, 'video')
|
||||
.withCommentTimestamp(8.2)
|
||||
const result = builder.build()
|
||||
|
||||
@@ -103,4 +150,18 @@ describe('AgentChatPromptBuilder', () => {
|
||||
expect(result).toContain('Attached Files:\n- file1.txt')
|
||||
expect(result).toContain('Referenced Workspace Assets:\n- folder1')
|
||||
})
|
||||
|
||||
it('should include location change context block in continuation mode when asset has changed', () => {
|
||||
const builder = new AgentChatPromptBuilder('a2')
|
||||
.withContinuation(true)
|
||||
.withAssetChanged(true)
|
||||
.withAssetDetails('new-folder', 'folder', undefined, undefined, 'folder')
|
||||
.withPathContext('projects/my-proj/new-folder')
|
||||
const result = builder.build()
|
||||
|
||||
expect(result).toContain('[Context: User Navigated to a New Location]')
|
||||
expect(result).toContain('New Location Asset ID: a2')
|
||||
expect(result).toContain('File Name: new-folder')
|
||||
expect(result).toContain('Asset Path Context:\nprojects/my-proj/new-folder')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,16 +1,7 @@
|
||||
export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'other' {
|
||||
if (!mediaType) return 'other'
|
||||
const lower = mediaType.toLowerCase()
|
||||
if (
|
||||
lower.startsWith('image/') ||
|
||||
lower === 'application/x-photoshop' ||
|
||||
lower === 'image/vnd.adobe.photoshop'
|
||||
) {
|
||||
return 'image'
|
||||
}
|
||||
if (lower.startsWith('video/')) {
|
||||
return 'video'
|
||||
}
|
||||
export function getSimpleMediaType(proxyType?: string | null): 'image' | 'video' | 'pdf' | 'other' {
|
||||
if (proxyType === 'image') return 'image'
|
||||
if (proxyType === 'video') return 'video'
|
||||
if (proxyType === 'pdf') return 'pdf'
|
||||
return 'other'
|
||||
}
|
||||
|
||||
@@ -18,7 +9,9 @@ export class AgentChatPromptBuilder {
|
||||
private assetId: string
|
||||
private assetName?: string
|
||||
private mediaType?: string
|
||||
private proxyType?: string
|
||||
private videoDuration?: number
|
||||
private totalPages?: number
|
||||
private commentTimestamp?: number
|
||||
private pathContext?: string
|
||||
private explicitMention = false
|
||||
@@ -26,6 +19,7 @@ export class AgentChatPromptBuilder {
|
||||
private referencedAssets: string[] = []
|
||||
|
||||
private isContinuation = false
|
||||
private hasAssetChanged = false
|
||||
|
||||
constructor(assetId: string) {
|
||||
this.assetId = assetId
|
||||
@@ -36,6 +30,11 @@ export class AgentChatPromptBuilder {
|
||||
return this
|
||||
}
|
||||
|
||||
withAssetChanged(hasAssetChanged?: boolean): this {
|
||||
this.hasAssetChanged = !!hasAssetChanged
|
||||
return this
|
||||
}
|
||||
|
||||
withAttachedFiles(files?: string[]): this {
|
||||
if (files) {
|
||||
this.attachedFiles = files
|
||||
@@ -55,10 +54,18 @@ export class AgentChatPromptBuilder {
|
||||
return this
|
||||
}
|
||||
|
||||
withAssetDetails(name: string, mediaType: string | null, duration?: number): this {
|
||||
withAssetDetails(
|
||||
name: string,
|
||||
mediaType: string | null,
|
||||
duration?: number,
|
||||
totalPages?: number,
|
||||
proxyType?: string | null,
|
||||
): this {
|
||||
this.assetName = name
|
||||
this.mediaType = mediaType || undefined
|
||||
this.proxyType = proxyType || undefined
|
||||
this.videoDuration = duration
|
||||
this.totalPages = totalPages
|
||||
return this
|
||||
}
|
||||
|
||||
@@ -75,13 +82,40 @@ export class AgentChatPromptBuilder {
|
||||
}
|
||||
|
||||
build(): string {
|
||||
const effectiveType = this.proxyType
|
||||
|
||||
if (this.isContinuation) {
|
||||
let instruction = ''
|
||||
if (this.hasAssetChanged) {
|
||||
instruction += `[Context: User Navigated to a New Location]\nNew Location Asset ID: ${this.assetId}`
|
||||
if (this.assetName) {
|
||||
instruction += `\nFile Name: ${this.assetName}`
|
||||
}
|
||||
if (effectiveType) {
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
instruction += `\nFile Type: ${type}`
|
||||
if (type === 'video' && this.videoDuration !== undefined) {
|
||||
instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds`
|
||||
} else if (type === 'pdf' && this.totalPages !== undefined) {
|
||||
instruction += `\nTotal Pages: ${this.totalPages}`
|
||||
}
|
||||
}
|
||||
if (this.pathContext) {
|
||||
instruction += `\n\nAsset Path Context:\n${this.pathContext}`
|
||||
}
|
||||
}
|
||||
if (this.commentTimestamp !== undefined) {
|
||||
instruction += `Comment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
if (instruction) instruction += '\n\n'
|
||||
if (type === 'pdf') {
|
||||
instruction += `Comment Page: ${Math.round(this.commentTimestamp)}`
|
||||
} else {
|
||||
instruction += `Comment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
|
||||
}
|
||||
}
|
||||
if (this.attachedFiles.length > 0 || this.referencedAssets.length > 0) {
|
||||
instruction += `\n\n[Context: New Attached Files & Referenced Assets]`
|
||||
if (instruction) instruction += '\n\n'
|
||||
instruction += `[Context: New Attached Files & Referenced Assets]`
|
||||
if (this.attachedFiles.length > 0) {
|
||||
instruction += `\nAttached Files:\n${this.attachedFiles.join('\n')}`
|
||||
}
|
||||
@@ -96,15 +130,22 @@ export class AgentChatPromptBuilder {
|
||||
if (this.assetName) {
|
||||
instruction += `\nFile Name: ${this.assetName}`
|
||||
}
|
||||
if (this.mediaType) {
|
||||
const type = getSimpleMediaType(this.mediaType)
|
||||
if (effectiveType) {
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
instruction += `\nFile Type: ${type}`
|
||||
if (type === 'video' && this.videoDuration !== undefined) {
|
||||
instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds`
|
||||
} else if (type === 'pdf' && this.totalPages !== undefined) {
|
||||
instruction += `\nTotal Pages: ${this.totalPages}`
|
||||
}
|
||||
}
|
||||
if (this.commentTimestamp !== undefined) {
|
||||
instruction += `\nComment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
if (type === 'pdf') {
|
||||
instruction += `\nComment Page: ${Math.round(this.commentTimestamp)}`
|
||||
} else {
|
||||
instruction += `\nComment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds`
|
||||
}
|
||||
}
|
||||
if (this.pathContext) {
|
||||
instruction += `\n\nAsset Path Context:\n${this.pathContext}`
|
||||
@@ -120,12 +161,14 @@ export class AgentChatPromptBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
if (this.mediaType) {
|
||||
const type = getSimpleMediaType(this.mediaType)
|
||||
if (effectiveType) {
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
if (type === 'image') {
|
||||
instruction += `\n\nIf you need to view the image data, call the 'analyze_image' tool. It does not require any parameters.`
|
||||
} else if (type === 'video') {
|
||||
instruction += `\n\nIf you need to view visual frames or take screenshots of the video, call the 'screenshot' tool. You must specify the 'start' (seconds), 'end' (seconds), and 'count' (number of screenshots) parameters.`
|
||||
} else if (type === 'pdf') {
|
||||
instruction += `\n\nIf you need to view pages of the PDF document, call the 'read_pdf_pages' tool. You must specify the 'start' (page number) and 'end' (page number) parameters. Maximum 20 pages allowed per call.`
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ describe('Agent Chat Workflow', () => {
|
||||
type: 'file',
|
||||
name: 'test-file.png',
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
parentId: 'parent-folder-id',
|
||||
})
|
||||
mockActivities.getCommentActivity.mockResolvedValue({
|
||||
@@ -133,7 +134,7 @@ describe('Agent Chat Workflow', () => {
|
||||
// Verify agent instruction composition
|
||||
const expectedInstruction1 = new AgentChatPromptBuilder('a1')
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined)
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
|
||||
.withCommentTimestamp(null)
|
||||
.withExplicitMention(true)
|
||||
.build()
|
||||
@@ -239,7 +240,7 @@ describe('Agent Chat Workflow', () => {
|
||||
// Verify instruction contains not explicitly mentioned instructions
|
||||
const expectedInstruction3 = new AgentChatPromptBuilder('a1')
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined)
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
|
||||
.withCommentTimestamp(null)
|
||||
.withExplicitMention(false)
|
||||
.build()
|
||||
@@ -271,18 +272,21 @@ describe('Agent Chat Workflow', () => {
|
||||
{
|
||||
asset: {
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
storageKey: { key: 'attachments/image1.png' },
|
||||
},
|
||||
},
|
||||
{
|
||||
asset: {
|
||||
mediaType: 'video/mp4', // non-image, should be ignored
|
||||
media: { proxyType: 'video' },
|
||||
storageKey: { key: 'attachments/video.mp4' },
|
||||
},
|
||||
},
|
||||
{
|
||||
asset: {
|
||||
mediaType: 'image/jpeg',
|
||||
media: { proxyType: 'image' },
|
||||
storageKey: { key: 'attachments/image2.jpg' },
|
||||
},
|
||||
},
|
||||
@@ -305,7 +309,7 @@ describe('Agent Chat Workflow', () => {
|
||||
|
||||
const expectedInstruction4 = new AgentChatPromptBuilder('a1')
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined)
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
|
||||
.withCommentTimestamp(null)
|
||||
.withExplicitMention(false)
|
||||
.build()
|
||||
@@ -512,4 +516,55 @@ describe('Agent Chat Workflow', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should handle version stack asset by building instructions with resolved latest version asset details', async () => {
|
||||
mockActivities.getAssetActivity.mockImplementation(async (id: string) => {
|
||||
if (id === 'vs-1') {
|
||||
return {
|
||||
id: 'v2-latest',
|
||||
project: { teamId: 't1' },
|
||||
type: 'file',
|
||||
name: 'latest-video.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
media: { proxyType: 'video', duration: 42 },
|
||||
parentId: 'vs-1',
|
||||
}
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: 'vs-1',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
agent: {
|
||||
prompt: 'Explain this version stack video',
|
||||
sessionId: 'session-vs-123',
|
||||
agentId: 'b1',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await agentChat(task)
|
||||
|
||||
const expectedInstruction = new AgentChatPromptBuilder('v2-latest')
|
||||
.withContinuation(true)
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('latest-video.mp4', 'video/mp4', 42, undefined, 'video')
|
||||
.build()
|
||||
|
||||
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
teamId: 't1',
|
||||
agentId: 'b1',
|
||||
message: 'Explain this version stack video',
|
||||
agentsInstruction: expectedInstruction,
|
||||
sessionId: 'session-vs-123',
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -74,7 +74,8 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
// 5. Prepare Images (Attachments only)
|
||||
if (userComment.attachments) {
|
||||
for (const att of userComment.attachments) {
|
||||
if (att.asset?.mediaType?.startsWith('image/') && att.asset?.storageKey?.key) {
|
||||
const attProxyType = (att.asset?.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (attProxyType === 'image' && att.asset?.storageKey?.key) {
|
||||
attachmentImageUrls.push(att.asset.storageKey.key)
|
||||
}
|
||||
}
|
||||
@@ -124,6 +125,7 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const mediaInfo = asset.media as any
|
||||
const duration = mediaInfo?.duration
|
||||
const totalPages = mediaInfo?.frames || mediaInfo?.metadata?.totalFrames
|
||||
|
||||
// Resolve Attached Files Details
|
||||
const attachedFileDetailsList: string[] = []
|
||||
@@ -160,10 +162,12 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const instruction = new AgentChatPromptBuilder(asset.id)
|
||||
.withContinuation(!isNewChat)
|
||||
.withAssetChanged(payload.agent?.hasAssetChanged)
|
||||
.withPathContext(pathContext)
|
||||
.withAssetDetails(asset.name, asset.mediaType, duration)
|
||||
.withAssetDetails(asset.name, asset.mediaType, duration, totalPages, proxyType)
|
||||
.withCommentTimestamp(commentTimestamp)
|
||||
.withExplicitMention(payload.agent?.explicitMention)
|
||||
.withAttachedFiles(attachedFileDetailsList)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -27,6 +27,7 @@ describe('Agent API', () => {
|
||||
model: 'model1',
|
||||
thinkingLevel: 'high',
|
||||
systemPrompt: 'prompt',
|
||||
deniedTools: ['bash'],
|
||||
},
|
||||
skills: [],
|
||||
},
|
||||
@@ -43,6 +44,7 @@ describe('Agent API', () => {
|
||||
expect(data).toHaveLength(1)
|
||||
expect(data[0].name).toBe('Bot 1')
|
||||
expect(data[0].providerId).toBe('prov1')
|
||||
expect(data[0].deniedTools).toEqual(['bash'])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -55,7 +57,13 @@ describe('Agent API', () => {
|
||||
providerId: 'prov1',
|
||||
modelId: 'model1',
|
||||
soul: 'New Soul',
|
||||
config: { provider: 'prov1', model: 'model1', thinkingLevel: 'low', systemPrompt: 'p' },
|
||||
config: {
|
||||
provider: 'prov1',
|
||||
model: 'model1',
|
||||
thinkingLevel: 'low',
|
||||
systemPrompt: 'p',
|
||||
deniedTools: ['bash'],
|
||||
},
|
||||
skills: [],
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
@@ -73,6 +81,7 @@ describe('Agent API', () => {
|
||||
systemPrompt: 'p',
|
||||
soul: 'New Soul',
|
||||
skills: ['skill1'],
|
||||
deniedTools: ['bash'],
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -80,6 +89,7 @@ describe('Agent API', () => {
|
||||
const data = await res.json()
|
||||
expect(data.name).toBe('New Agent')
|
||||
expect(data.soul).toBe('New Soul')
|
||||
expect(data.deniedTools).toEqual(['bash'])
|
||||
})
|
||||
|
||||
it('validates request', async () => {
|
||||
@@ -105,7 +115,7 @@ describe('Agent API', () => {
|
||||
providerId: 'prov1',
|
||||
modelId: 'model1',
|
||||
soul: 'Updated Soul',
|
||||
config: { provider: 'prov1', model: 'model1' },
|
||||
config: { provider: 'prov1', model: 'model1', deniedTools: ['bash'] },
|
||||
skills: [],
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
@@ -121,6 +131,7 @@ describe('Agent API', () => {
|
||||
modelId: 'model1',
|
||||
soul: 'Updated Soul',
|
||||
skills: [],
|
||||
deniedTools: ['bash'],
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -128,6 +139,7 @@ describe('Agent API', () => {
|
||||
const data = await res.json()
|
||||
expect(data.name).toBe('Updated Agent')
|
||||
expect(data.soul).toBe('Updated Soul')
|
||||
expect(data.deniedTools).toEqual(['bash'])
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
skillId: s.skillId,
|
||||
skill: s.skill,
|
||||
})),
|
||||
deniedTools: config.deniedTools || [],
|
||||
}
|
||||
}),
|
||||
)
|
||||
@@ -89,6 +90,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
skillId: s.skillId,
|
||||
skill: s.skill,
|
||||
})),
|
||||
deniedTools: config.deniedTools || [],
|
||||
}
|
||||
|
||||
return c.json(info, 200)
|
||||
@@ -127,6 +129,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
skillId: s.skillId,
|
||||
skill: s.skill,
|
||||
})),
|
||||
deniedTools: config.deniedTools || [],
|
||||
}
|
||||
|
||||
return c.json(info, 200)
|
||||
|
||||
@@ -57,6 +57,7 @@ describe('file api', () => {
|
||||
vi.spyOn(assetService, 'deleteAssets').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'createComment').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'completeComment').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'deleteComment').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'listComments').mockImplementation(vi.fn())
|
||||
vi.spyOn(assetService, 'restoreAssets').mockImplementation(vi.fn())
|
||||
vi.spyOn(metadataService, 'updateAssetMetadata').mockImplementation(vi.fn())
|
||||
@@ -78,7 +79,7 @@ describe('file api', () => {
|
||||
fileCount: 1,
|
||||
type: 'file',
|
||||
status: 'active',
|
||||
mediaType: 'image/jpeg',
|
||||
proxyType: 'image',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
@@ -106,7 +107,7 @@ describe('file api', () => {
|
||||
fileCount: 1,
|
||||
type: 'file',
|
||||
status: 'active',
|
||||
mediaType: 'image/jpeg',
|
||||
proxyType: 'image',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
@@ -358,6 +359,30 @@ describe('file api', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('DELETE /comments/:commentId', async () => {
|
||||
vi.mocked(assetService.deleteComment).mockResolvedValue(undefined)
|
||||
|
||||
const app = new Hono().use('*', authMiddleware).route('/', fileRoute)
|
||||
const res = await app.request('/comments/comment-id', {
|
||||
method: 'DELETE',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const json = await res.json()
|
||||
expect(json.success).toBe(true)
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user1', name: 'Test User' },
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Comment,
|
||||
id: 'comment-id',
|
||||
})
|
||||
expect(assetService.deleteComment).toHaveBeenCalledWith({
|
||||
commentId: 'comment-id',
|
||||
userId: 'user1',
|
||||
})
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/files - image with compression', async () => {
|
||||
vi.mocked(s3Service.putObject).mockResolvedValue(undefined)
|
||||
|
||||
@@ -475,7 +500,7 @@ describe('file api', () => {
|
||||
fileCount: 1,
|
||||
type: 'file',
|
||||
status: 'active',
|
||||
mediaType: 'image/jpeg',
|
||||
proxyType: 'image',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
|
||||
@@ -220,6 +220,20 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
return c.json(updated)
|
||||
},
|
||||
)
|
||||
.delete('/comments/:commentId', async (c) => {
|
||||
const commentId = c.req.param('commentId')
|
||||
const user = c.get('user')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Comment,
|
||||
id: commentId,
|
||||
})
|
||||
|
||||
await assetService.deleteComment({ commentId, userId: user.id })
|
||||
return c.json({ success: true })
|
||||
})
|
||||
.post('/files/restore', zValidator('json', restoreFilesRequestSchema), async (c) => {
|
||||
const user = c.get('user')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
@@ -53,7 +53,6 @@ describe('folder api', () => {
|
||||
fileCount: 1,
|
||||
type: 'folder',
|
||||
status: 'active',
|
||||
mediaType: null,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
@@ -81,7 +80,6 @@ describe('folder api', () => {
|
||||
fileCount: 1,
|
||||
type: 'folder',
|
||||
status: 'active',
|
||||
mediaType: null,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
@@ -113,7 +111,6 @@ describe('folder api', () => {
|
||||
fileCount: 0,
|
||||
type: 'folder',
|
||||
status: 'active',
|
||||
mediaType: null,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
@@ -147,7 +144,6 @@ describe('folder api', () => {
|
||||
fileCount: 0,
|
||||
type: 'folder',
|
||||
status: 'active',
|
||||
mediaType: null,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
@@ -223,7 +219,6 @@ describe('folder api', () => {
|
||||
fileCount: 0,
|
||||
type: 'folder',
|
||||
status: 'active',
|
||||
mediaType: null,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
@@ -261,7 +256,6 @@ describe('folder api', () => {
|
||||
fileCount: 1,
|
||||
type: 'folder',
|
||||
status: 'active',
|
||||
mediaType: null,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
|
||||
@@ -1,23 +1,25 @@
|
||||
{
|
||||
"name": "@shumai/core",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@sinclair/typebox": "0.34.49",
|
||||
"adm-zip": "0.5.17",
|
||||
"better-auth": "1.6.11",
|
||||
"csv-parse": "7.0.1",
|
||||
"date-fns": "4.4.0",
|
||||
"dotenv": "16.6.1",
|
||||
"dotenv-expand": "13.0.0",
|
||||
"extract-zip": "2.0.1",
|
||||
"hono": "4.12.23",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"pdfkit": "0.19.1",
|
||||
"pino": "10.3.1",
|
||||
"sharp": "0.34.5",
|
||||
"tweetnacl": "1.0.3",
|
||||
@@ -27,6 +29,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/adm-zip": "0.5.8",
|
||||
"@types/extract-zip": "2.0.3"
|
||||
"@types/extract-zip": "2.0.3",
|
||||
"@types/pdfkit": "0.17.6"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,6 +126,27 @@ describe('AgentService', () => {
|
||||
expect(agent?.skills?.[0].skillId).toBe(skill.id)
|
||||
})
|
||||
|
||||
test('Create Agent with Denied Tools', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team, provider, model } = await setupTestData(db)
|
||||
|
||||
const agent = await svc.createAgent({
|
||||
teamId: team.id,
|
||||
name: 'Denied Tools Agent',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
thinkingLevel: 'medium',
|
||||
deniedTools: ['bash', 'screenshot'],
|
||||
})
|
||||
|
||||
expect(agent?.id).toBeDefined()
|
||||
const config = agent?.config as unknown as PrismaJson.AgentConfig
|
||||
expect(config.deniedTools).toEqual(['bash', 'screenshot'])
|
||||
})
|
||||
|
||||
test('Create Agent validation fails if model mismatch', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
@@ -186,8 +207,11 @@ describe('AgentService', () => {
|
||||
thinkingLevel: 'high',
|
||||
soul: 'New soul',
|
||||
skills: [skill.id], // Re-enable one
|
||||
deniedTools: ['list_assets'],
|
||||
})
|
||||
expect(reUpdated.skills?.length).toBe(1)
|
||||
const config = reUpdated.config as unknown as PrismaJson.AgentConfig
|
||||
expect(config.deniedTools).toEqual(['list_assets'])
|
||||
})
|
||||
|
||||
test('Create Agent with presigned URL extracts S3 key', async () => {
|
||||
|
||||
@@ -114,6 +114,7 @@ export class AgentService {
|
||||
systemPrompt,
|
||||
soul,
|
||||
skills,
|
||||
deniedTools,
|
||||
} = params
|
||||
|
||||
if (type === 'embedding' || type === 'autofill') {
|
||||
@@ -152,6 +153,7 @@ export class AgentService {
|
||||
model: modelId || '',
|
||||
thinkingLevel,
|
||||
systemPrompt,
|
||||
deniedTools: deniedTools || [],
|
||||
}
|
||||
|
||||
const user = await tx.user.create({
|
||||
@@ -211,6 +213,7 @@ export class AgentService {
|
||||
systemPrompt,
|
||||
soul,
|
||||
skills,
|
||||
deniedTools,
|
||||
} = params
|
||||
|
||||
const agent = await this.prismaClient.agent.findUnique({
|
||||
@@ -251,6 +254,7 @@ export class AgentService {
|
||||
model: modelId || '',
|
||||
thinkingLevel,
|
||||
systemPrompt,
|
||||
deniedTools: deniedTools || [],
|
||||
}
|
||||
|
||||
return this.prismaClient.$transaction(async (tx) => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
|
||||
import { AssetType, AssetStatus, Prisma } from '@shumai/db'
|
||||
import { AssetService } from './asset'
|
||||
@@ -721,6 +722,161 @@ describe('AssetService', () => {
|
||||
expect(list.data[0].replies.length).toBe(1)
|
||||
})
|
||||
|
||||
it('deleteComment validation and execution', async () => {
|
||||
const { user, project, assets } = await setupBasicAssets()
|
||||
|
||||
// 1. Create a comment
|
||||
const comment = await assetService.createComment({
|
||||
assetId: assets.fileA1.id,
|
||||
userId: user.id,
|
||||
message: 'Self Comment',
|
||||
attachmentIds: [],
|
||||
})
|
||||
|
||||
// 2. Add user as a team member with owner role (team owner)
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: project.teamId,
|
||||
userId: user.id,
|
||||
role: 'owner',
|
||||
},
|
||||
})
|
||||
|
||||
// Create another user
|
||||
const otherUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'OtherUser',
|
||||
type: 'human',
|
||||
email: `other-${Date.now()}@example.com`,
|
||||
},
|
||||
})
|
||||
|
||||
// Add other user to the team as editor
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: project.teamId,
|
||||
userId: otherUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
// Create comment by other user
|
||||
const otherComment = await assetService.createComment({
|
||||
assetId: assets.fileA1.id,
|
||||
userId: otherUser.id,
|
||||
message: 'Other Comment',
|
||||
attachmentIds: [],
|
||||
})
|
||||
|
||||
// 3. User (owner) should be able to delete their own comment
|
||||
await expect(
|
||||
assetService.deleteComment({ commentId: comment.id, userId: user.id }),
|
||||
).resolves.not.toThrow()
|
||||
// Verify it is deleted from DB
|
||||
await expect(prisma.assetComment.findUnique({ where: { id: comment.id } })).resolves.toBeNull()
|
||||
|
||||
// 4. User (owner) should be able to delete other user's comment
|
||||
await expect(
|
||||
assetService.deleteComment({ commentId: otherComment.id, userId: user.id }),
|
||||
).resolves.not.toThrow()
|
||||
// Verify it is deleted from DB
|
||||
await expect(
|
||||
prisma.assetComment.findUnique({ where: { id: otherComment.id } }),
|
||||
).resolves.toBeNull()
|
||||
|
||||
// Create a new comment by user
|
||||
const comment2 = await assetService.createComment({
|
||||
assetId: assets.fileA1.id,
|
||||
userId: user.id,
|
||||
message: 'Self Comment 2',
|
||||
attachmentIds: [],
|
||||
})
|
||||
|
||||
// 5. Other user (editor, non-creator) should NOT be able to delete user's comment
|
||||
await expect(
|
||||
assetService.deleteComment({ commentId: comment2.id, userId: otherUser.id }),
|
||||
).rejects.toThrow(HTTPException)
|
||||
|
||||
// 6. Other user (editor, creator) should be able to delete their own comment
|
||||
const otherComment2 = await assetService.createComment({
|
||||
assetId: assets.fileA1.id,
|
||||
userId: otherUser.id,
|
||||
message: 'Other Comment 2',
|
||||
attachmentIds: [],
|
||||
})
|
||||
await expect(
|
||||
assetService.deleteComment({ commentId: otherComment2.id, userId: otherUser.id }),
|
||||
).resolves.not.toThrow()
|
||||
})
|
||||
|
||||
it('deleteComment cleans up S3 attachment files and database records', async () => {
|
||||
const { user, project, assets } = await setupBasicAssets()
|
||||
|
||||
// 1. Add user as team owner
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: project.teamId,
|
||||
userId: user.id,
|
||||
role: 'owner',
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Create StorageKey and Asset for attachment
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'attachments/test-attachment-file.png',
|
||||
status: 'active',
|
||||
},
|
||||
})
|
||||
|
||||
const attachmentAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test-attachment-file.png',
|
||||
type: AssetType.file,
|
||||
projectId: project.id,
|
||||
parentId: assets.root.id,
|
||||
creatorId: user.id,
|
||||
storageKeyId: storageKey.id,
|
||||
sizeByte: 100,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
// 3. Create a comment with this attachment
|
||||
const comment = await assetService.createComment({
|
||||
assetId: assets.fileA1.id,
|
||||
userId: user.id,
|
||||
message: 'Comment with attachment',
|
||||
attachmentIds: [attachmentAsset.id],
|
||||
})
|
||||
|
||||
// Verify it is created in DB
|
||||
const dbComment = await prisma.assetComment.findUnique({
|
||||
where: { id: comment.id },
|
||||
include: { attachments: true },
|
||||
})
|
||||
expect(dbComment?.attachments.length).toBe(1)
|
||||
|
||||
// 4. Delete the comment
|
||||
await expect(
|
||||
assetService.deleteComment({ commentId: comment.id, userId: user.id }),
|
||||
).resolves.not.toThrow()
|
||||
|
||||
// 5. Verify the comment is deleted from DB
|
||||
await expect(prisma.assetComment.findUnique({ where: { id: comment.id } })).resolves.toBeNull()
|
||||
|
||||
// 6. Verify that the attachment Asset and StorageKey are deleted from DB
|
||||
await expect(prisma.asset.findUnique({ where: { id: attachmentAsset.id } })).resolves.toBeNull()
|
||||
await expect(prisma.storageKey.findUnique({ where: { id: storageKey.id } })).resolves.toBeNull()
|
||||
|
||||
// 7. Verify S3 deletion was called
|
||||
const { s3Service } = await import('@shumai/core/src/s3/s3')
|
||||
expect(s3Service.deleteObject).toHaveBeenCalledWith(
|
||||
'shumai',
|
||||
'attachments/test-attachment-file.png',
|
||||
)
|
||||
})
|
||||
|
||||
it('can create and retrieve comments with video timestamp', async () => {
|
||||
const { user, assets } = await setupBasicAssets()
|
||||
|
||||
@@ -1319,6 +1475,7 @@ describe('AssetService', () => {
|
||||
mediaType: 'video/mp4',
|
||||
status: 'processed',
|
||||
media: {
|
||||
proxyType: 'video',
|
||||
poster: { key: 'poster-key' },
|
||||
metadata: { duration: 262 },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
@@ -1329,7 +1486,7 @@ describe('AssetService', () => {
|
||||
const info = await assetService.getAsset({ assetId: video.id })
|
||||
|
||||
expect(info.preview).toBeDefined()
|
||||
expect(info.preview?.mediaType).toBe('video/mp4')
|
||||
expect(info.preview?.proxyType).toBe('video')
|
||||
expect(info.preview?.duration).toBe(262)
|
||||
})
|
||||
|
||||
@@ -1346,6 +1503,7 @@ describe('AssetService', () => {
|
||||
mediaType: 'image/png',
|
||||
status: 'processed',
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
thumbnail: { key: 'thumb-key' },
|
||||
metadata: { originalWidth: 100, originalHeight: 80 },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
@@ -1374,6 +1532,7 @@ describe('AssetService', () => {
|
||||
mediaType: 'image/png',
|
||||
status: 'processed',
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
thumbnail: { key: 'thumb-key' },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
@@ -1398,7 +1557,7 @@ describe('AssetService', () => {
|
||||
expect(info.type).toBe(AssetType.symlink)
|
||||
// Resolved from target
|
||||
expect(info.sizeByte).toBe(5000)
|
||||
expect(info.mediaType).toBe('image/png')
|
||||
expect(info.proxyType).toBe('image')
|
||||
expect(info.media).toBeDefined()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((info.media as any).thumbnail?.key).toBe('thumb-key')
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
AssetUserInfo,
|
||||
} from '@shumai/dtos'
|
||||
import { type Asset, AssetStatus, AssetType, Prisma, type StorageKey } from '@shumai/db'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { PaginatedData, paginateQuery, PaginationParams } from '@shumai/core/src/pagination'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
@@ -101,6 +102,7 @@ export class AssetService {
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -108,6 +110,7 @@ export class AssetService {
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -660,6 +663,7 @@ export class AssetService {
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -667,6 +671,7 @@ export class AssetService {
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -744,6 +749,7 @@ export class AssetService {
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -751,6 +757,7 @@ export class AssetService {
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -793,6 +800,7 @@ export class AssetService {
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -800,6 +808,7 @@ export class AssetService {
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -935,6 +944,7 @@ export class AssetService {
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -942,6 +952,7 @@ export class AssetService {
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
@@ -1413,6 +1424,91 @@ export class AssetService {
|
||||
return this.getComment(commentId)
|
||||
}
|
||||
|
||||
async deleteComment({ commentId, userId }: { commentId: string; userId: string }): Promise<void> {
|
||||
const comment = await this.prismaClient.assetComment.findUnique({
|
||||
where: { id: commentId },
|
||||
include: {
|
||||
attachments: {
|
||||
include: {
|
||||
asset: {
|
||||
include: {
|
||||
storageKey: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
asset: {
|
||||
select: {
|
||||
project: { select: { id: true, teamId: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if (!comment) {
|
||||
throw new HTTPException(404, { message: 'Comment not found' })
|
||||
}
|
||||
|
||||
const teamId = comment.asset.project?.teamId
|
||||
|
||||
if (!teamId) {
|
||||
throw new HTTPException(400, {
|
||||
message: 'Data corruption: Comment asset is not associated with a project',
|
||||
})
|
||||
}
|
||||
|
||||
const member = await this.prismaClient.teamMember.findUnique({
|
||||
where: {
|
||||
teamIdUserId: {
|
||||
teamId,
|
||||
userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if (!member) {
|
||||
throw new HTTPException(403, { message: 'User is not a member of the team' })
|
||||
}
|
||||
|
||||
const isOwner = member.role === 'owner'
|
||||
const isCreator = comment.creatorId === userId
|
||||
|
||||
if (!isOwner && !isCreator) {
|
||||
throw new HTTPException(403, { message: 'You do not have permission to delete this comment' })
|
||||
}
|
||||
|
||||
// Clean up attachment files from S3/storage and delete their DB records
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
for (const att of comment.attachments) {
|
||||
const asset = att.asset
|
||||
if (asset.storageKey) {
|
||||
try {
|
||||
await s3Service.deleteObject(bucket, asset.storageKey.key)
|
||||
} catch (err) {
|
||||
logger.error({ err, assetId: asset.id }, 'Failed to delete attachment file from S3')
|
||||
}
|
||||
}
|
||||
|
||||
await this.prismaClient.asset.delete({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
|
||||
if (asset.storageKey) {
|
||||
await this.prismaClient.storageKey
|
||||
.delete({
|
||||
where: { id: asset.storageKey.id },
|
||||
})
|
||||
.catch(() => {
|
||||
// Silently ignore if already deleted
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
await this.prismaClient.assetComment.delete({
|
||||
where: { id: commentId },
|
||||
})
|
||||
}
|
||||
|
||||
async listComments(
|
||||
assetId: string,
|
||||
params: PaginationParams,
|
||||
@@ -1555,6 +1651,7 @@ export class AssetService {
|
||||
folderForPreview.children
|
||||
) {
|
||||
for (const child of folderForPreview.children) {
|
||||
if (child.isDeleted) continue
|
||||
let previewAsset = child as Asset
|
||||
let mediaType = child.mediaType
|
||||
|
||||
@@ -1611,6 +1708,13 @@ export class AssetService {
|
||||
'GET',
|
||||
)
|
||||
}
|
||||
if (media && media.pdfTranscode?.key) {
|
||||
media.pdfTranscode.url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
media.pdfTranscode.key,
|
||||
'GET',
|
||||
)
|
||||
}
|
||||
|
||||
const key = latestVersion.storageKey?.key
|
||||
if (key) {
|
||||
@@ -1629,6 +1733,8 @@ export class AssetService {
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
|
||||
result.push({
|
||||
id: a.id,
|
||||
name:
|
||||
@@ -1640,7 +1746,7 @@ export class AssetService {
|
||||
type: a.type,
|
||||
targetType: a.type === AssetType.symlink ? a.target?.type : null,
|
||||
status: latestVersion.status,
|
||||
mediaType: latestVersion.mediaType,
|
||||
proxyType,
|
||||
latestChildren,
|
||||
preview,
|
||||
createdAt: a.createdAt.toISOString(),
|
||||
@@ -1904,14 +2010,16 @@ export class AssetService {
|
||||
private async toPreviewInfo(asset: Asset | AssetWithIncludes): Promise<PreviewInfo | null> {
|
||||
if (!asset.media) return null
|
||||
|
||||
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
|
||||
let thumbnailUrl = undefined
|
||||
if (asset.mediaType?.startsWith('image/') && asset.media.thumbnail?.key) {
|
||||
if (proxyType === 'image' && asset.media.thumbnail?.key) {
|
||||
thumbnailUrl = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
asset.media.thumbnail.key,
|
||||
'GET',
|
||||
)
|
||||
} else if (asset.mediaType?.startsWith('video/') && asset.media.poster?.key) {
|
||||
} else if (asset.media.poster?.key) {
|
||||
thumbnailUrl = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
asset.media.poster.key,
|
||||
@@ -1929,12 +2037,13 @@ export class AssetService {
|
||||
}
|
||||
|
||||
return {
|
||||
mediaType: asset.mediaType,
|
||||
proxyType,
|
||||
thumbnailUrl,
|
||||
originalHeight: asset.media.metadata?.originalHeight,
|
||||
originalWidth: asset.media.metadata?.originalWidth,
|
||||
spriteUrl,
|
||||
duration: asset.media.metadata?.duration,
|
||||
pageCount: asset.media.metadata?.totalFrames,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1957,11 +2066,17 @@ export class AssetService {
|
||||
a.asset.storageKey.key,
|
||||
'GET',
|
||||
)
|
||||
const attachmentProxyType = (a.asset.media?.proxyType || null) as
|
||||
| 'image'
|
||||
| 'video'
|
||||
| 'audio'
|
||||
| 'pdf'
|
||||
| null
|
||||
attachments.push({
|
||||
id: a.id,
|
||||
assetId: a.asset.id,
|
||||
url,
|
||||
mediaType: a.asset.mediaType,
|
||||
proxyType: attachmentProxyType,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,6 +155,51 @@ describe('ChatService', () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const taskPayload = task?.payload as any
|
||||
expect(taskPayload?.agent?.prompt).toBe('second message')
|
||||
expect(taskPayload?.agent?.hasAssetChanged).toBe(false)
|
||||
})
|
||||
|
||||
it('should update session assetId and set hasAssetChanged when continuing chat with a new contextAssetId', async () => {
|
||||
const { user, team, project, rootFolder } = await setupBasicData()
|
||||
|
||||
const newFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'new_folder',
|
||||
type: 'folder',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Start session on rootFolder
|
||||
const { sessionId } = await chatService.startOrContinueChat(user, team.id, {
|
||||
agentId: 'test-agent-id',
|
||||
textPrompt: 'message in root folder',
|
||||
projectId: project.id,
|
||||
contextAssetId: rootFolder.id,
|
||||
})
|
||||
|
||||
const sessionBefore = await prisma.agentSession.findUnique({ where: { id: sessionId } })
|
||||
expect(sessionBefore?.assetId).toBe(rootFolder.id)
|
||||
|
||||
// Continue session on newFolder
|
||||
const { taskId: nextTaskId } = await chatService.startOrContinueChat(user, team.id, {
|
||||
agentId: 'test-agent-id',
|
||||
textPrompt: 'message after switching to new folder',
|
||||
sessionId,
|
||||
contextAssetId: newFolder.id,
|
||||
})
|
||||
|
||||
// Verify session assetId updated
|
||||
const sessionAfter = await prisma.agentSession.findUnique({ where: { id: sessionId } })
|
||||
expect(sessionAfter?.assetId).toBe(newFolder.id)
|
||||
|
||||
// Verify task payload has assetId set to newFolder.id and hasAssetChanged is true
|
||||
const task = await prisma.workflowTask.findUnique({ where: { id: nextTaskId } })
|
||||
expect(task?.assetId).toBe(newFolder.id)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const taskPayload = task?.payload as any
|
||||
expect(taskPayload?.agent?.hasAssetChanged).toBe(true)
|
||||
})
|
||||
|
||||
it('should inject context of referenced assets and attached files', async () => {
|
||||
@@ -181,6 +226,8 @@ describe('ChatService', () => {
|
||||
name: 'doc.jpg',
|
||||
type: 'file',
|
||||
mediaType: 'image/jpeg',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
|
||||
media: { proxyType: 'image' } as any,
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
parentId: childFolder.id,
|
||||
|
||||
@@ -185,6 +185,7 @@ export class ChatService {
|
||||
|
||||
// 2. Project / Asset scoping resolution
|
||||
let targetAssetId = ''
|
||||
let hasAssetChanged = false
|
||||
|
||||
if (passedSessionId) {
|
||||
const session = await this.prismaClient.agentSession.findUnique({
|
||||
@@ -194,7 +195,18 @@ export class ChatService {
|
||||
if (!session) throw new Error('Session not found')
|
||||
if (session.userId !== user.id) throw new Error('Unauthorized session access')
|
||||
if (!session.assetId) throw new Error('Session has no associated asset')
|
||||
targetAssetId = session.assetId
|
||||
|
||||
if (contextAssetId && contextAssetId !== session.assetId) {
|
||||
const contextAsset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: contextAssetId },
|
||||
include: { project: true },
|
||||
})
|
||||
if (!contextAsset) throw new Error('Context asset not found')
|
||||
targetAssetId = contextAssetId
|
||||
hasAssetChanged = true
|
||||
} else {
|
||||
targetAssetId = session.assetId
|
||||
}
|
||||
} else {
|
||||
// Start a new session
|
||||
if (contextAssetId) {
|
||||
@@ -232,7 +244,8 @@ export class ChatService {
|
||||
where: { id: fileId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (file && file.mediaType?.startsWith('image/') && file.storageKey?.key) {
|
||||
const isImage = (file?.media as PrismaJson.MediaInfo | null)?.proxyType === 'image'
|
||||
if (file && isImage && file.storageKey?.key) {
|
||||
resolvedImageUrls.push(file.storageKey.key)
|
||||
}
|
||||
}
|
||||
@@ -268,10 +281,13 @@ export class ChatService {
|
||||
if (!sessionExists) throw new Error('Session not found')
|
||||
if (sessionExists.userId !== user.id) throw new Error('Unauthorized session access')
|
||||
|
||||
// Update the session's agentId to match the selected agent
|
||||
// Update the session's agentId and assetId (if location changed)
|
||||
await tx.agentSession.update({
|
||||
where: { id: activeSessionId },
|
||||
data: { agentId },
|
||||
data: {
|
||||
agentId,
|
||||
...(hasAssetChanged ? { assetId: targetAssetId } : {}),
|
||||
},
|
||||
})
|
||||
} else {
|
||||
const newSession = await tx.agentSession.create({
|
||||
@@ -306,6 +322,7 @@ export class ChatService {
|
||||
sessionId: activeSessionId,
|
||||
userId: user.id,
|
||||
isNewChat: !passedSessionId,
|
||||
hasAssetChanged,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -112,6 +112,23 @@ export const systemFields: Prisma.MetadataFieldCreateInput[] = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
key: 'proxy_type',
|
||||
scope: 'SYSTEM',
|
||||
readOnly: true,
|
||||
config: {
|
||||
name: 'Proxy Type',
|
||||
type: 'select',
|
||||
select: {
|
||||
options: [
|
||||
{ id: 'image', displayName: 'Image', color: 'system' },
|
||||
{ id: 'video', displayName: 'Video', color: 'system' },
|
||||
{ id: 'audio', displayName: 'Audio', color: 'system' },
|
||||
{ id: 'pdf', displayName: 'PDF', color: 'system' },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
key: 'audio_bit_depth',
|
||||
scope: 'SYSTEM',
|
||||
|
||||
@@ -401,7 +401,7 @@ export class NotificationService {
|
||||
info.asset = {
|
||||
id: a.id,
|
||||
name: a.name,
|
||||
mediaType: a.mediaType || undefined,
|
||||
proxyType: (a.media as PrismaJson.MediaInfo | null)?.proxyType || undefined,
|
||||
preview: previewUrl || undefined,
|
||||
thumbnailUrl: thumbnailUrl || undefined,
|
||||
originalWidth: originalWidth || undefined,
|
||||
|
||||
@@ -7,6 +7,7 @@ import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
|
||||
import { AssetType } from '@shumai/db'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
|
||||
describe('SearchService', () => {
|
||||
setupTestDbHooks()
|
||||
@@ -559,6 +560,57 @@ describe('SearchService', () => {
|
||||
expect(resultWithSymlink.data.find((a) => a.id === symlink.id)).toBeDefined()
|
||||
})
|
||||
|
||||
it('excludes soft-deleted (trashed) files from latestChildren when searching folders', async () => {
|
||||
const { rootFolder } = await setupBasicAssets()
|
||||
|
||||
const subfolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'subfolder',
|
||||
type: AssetType.folder,
|
||||
projectId: rootFolder.projectId,
|
||||
parentId: rootFolder.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const childFile1 = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'child1.txt',
|
||||
type: AssetType.file,
|
||||
projectId: rootFolder.projectId,
|
||||
parentId: subfolder.id,
|
||||
status: 'uploaded',
|
||||
sortIndex: 'a0',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'child2.txt',
|
||||
type: AssetType.file,
|
||||
projectId: rootFolder.projectId,
|
||||
parentId: subfolder.id,
|
||||
status: 'uploaded',
|
||||
sortIndex: 'a1',
|
||||
},
|
||||
})
|
||||
|
||||
await assetService.deleteAssets([childFile1.id])
|
||||
|
||||
const result = await searchService.search(rootFolder.id, {
|
||||
recursively: false,
|
||||
assetType: 'folder',
|
||||
operator: 'AND',
|
||||
conditions: [],
|
||||
isSemantic: false,
|
||||
})
|
||||
|
||||
const searchedSubfolder = result.data.find((a) => a.id === subfolder.id)
|
||||
expect(searchedSubfolder).toBeDefined()
|
||||
|
||||
expect(searchedSubfolder?.latestChildren?.length).toBe(1)
|
||||
})
|
||||
|
||||
describe('Semantic Search with multiple chunks and distance ordering', () => {
|
||||
it('returns multiple matching segments from the same asset and orders by distance', async () => {
|
||||
const user = await prisma.user.create({
|
||||
|
||||
@@ -1,11 +1,20 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { transcodeService } from './transcode'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import * as path from 'path'
|
||||
import * as child_process from 'child_process'
|
||||
import * as fs from 'fs'
|
||||
import { WorkflowTask } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import sharp from 'sharp'
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
downloadToFile: vi.fn(),
|
||||
putObject: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
// Mock child_process
|
||||
vi.mock('child_process', () => ({
|
||||
execFile: vi.fn(),
|
||||
@@ -16,7 +25,12 @@ vi.mock('sharp', () => {
|
||||
const mockSharp = {
|
||||
resize: vi.fn().mockReturnThis(),
|
||||
webp: vi.fn().mockReturnThis(),
|
||||
toFile: vi.fn().mockResolvedValue({}),
|
||||
composite: vi.fn().mockReturnThis(),
|
||||
toBuffer: vi.fn().mockResolvedValue(Buffer.from('fake-webp-buffer')),
|
||||
toFile: vi.fn().mockImplementation(async (filePath: string) => {
|
||||
fs.writeFileSync(filePath, 'fake-webp-data')
|
||||
return {}
|
||||
}),
|
||||
metadata: vi.fn().mockResolvedValue({ width: 800, height: 600, format: 'png' }),
|
||||
}
|
||||
const sharpFunc = vi.fn(() => mockSharp)
|
||||
@@ -265,7 +279,7 @@ describe('TranscodeService', () => {
|
||||
})) as WorkflowTask
|
||||
|
||||
expect(task.assetId).toBe('asset-123')
|
||||
expect(task.type).toBe('transcode')
|
||||
expect(task.type).toBe('transcode_video')
|
||||
expect(task.status).toBe('pending')
|
||||
expect(task.payload?.projectId).toBe('proj-123')
|
||||
expect(task.payload?.transcode?.videoStrategy).toBe('best_match')
|
||||
@@ -347,4 +361,268 @@ describe('TranscodeService', () => {
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate PDF sprite sheet and poster using pdftoppm, ffmpeg, and sharp', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
cmd: string,
|
||||
args: string[],
|
||||
cb: (err: Error | null, res: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
if (cmd === 'pdftoppm') {
|
||||
// Create dummy page-1.png in temp directory (target output path prefix is args[args.length - 1])
|
||||
const pagePrefix = args[args.length - 1]
|
||||
const pagePath = `${pagePrefix}-1.png`
|
||||
fs.writeFileSync(pagePath, 'fake-png-data')
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
} else if (cmd === 'pdfinfo') {
|
||||
cb(null, { stdout: 'Title: Document\nPages: 15\nPage size: 612 x 792 pts', stderr: '' })
|
||||
} else if (cmd === 'ffmpeg') {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
} else {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
const outputSprite = path.join(tempDir, 'sprite.webp')
|
||||
const outputPoster = path.join(tempDir, 'poster.webp')
|
||||
|
||||
const result = await transcodeService.generatePdfSprite('input.pdf', outputSprite, outputPoster)
|
||||
|
||||
expect(result.pageCount).toBe(15)
|
||||
expect(result.originalWidth).toBe(800)
|
||||
expect(result.originalHeight).toBe(600)
|
||||
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'pdftoppm',
|
||||
expect.arrayContaining(['-png', '-f', '1', '-l', '100', 'input.pdf']),
|
||||
expect.any(Function),
|
||||
)
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'ffmpeg',
|
||||
expect.arrayContaining(['-filter_complex', 'scale=w=480:h=-2,tile=10x10']),
|
||||
expect.any(Function),
|
||||
)
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'pdfinfo',
|
||||
['input.pdf'],
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
|
||||
it('should extract PDF info metadata using pdftoppm and pdfinfo', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
cmd: string,
|
||||
args: string[],
|
||||
cb: (err: Error | null, res: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
if (cmd === 'pdftoppm') {
|
||||
const pagePrefix = args[args.length - 1]
|
||||
const pagePath = `${pagePrefix}-1.png`
|
||||
fs.writeFileSync(pagePath, 'fake-png-data')
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
} else if (cmd === 'pdfinfo') {
|
||||
cb(null, { stdout: 'Pages: 42\n', stderr: '' })
|
||||
} else {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
const info = await transcodeService.getPdfInfo('document.pdf')
|
||||
|
||||
expect(info.mimeType).toBe('application/pdf')
|
||||
expect(info.totalFrames).toBe(42)
|
||||
expect(info.originalWidth).toBe(800)
|
||||
expect(info.originalHeight).toBe(600)
|
||||
expect(info.hasAudio).toBe(false)
|
||||
})
|
||||
|
||||
it('should render PDF pages and upload to S3', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
cmd: string,
|
||||
args: string[],
|
||||
cb: (err: Error | null, res: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
if (cmd === 'pdftoppm') {
|
||||
const pagePrefix = args[args.length - 1]
|
||||
const pagePath = `${pagePrefix}-1.png`
|
||||
fs.writeFileSync(pagePath, 'fake-png-data')
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
} else {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
vi.mocked(s3Service.downloadToFile).mockResolvedValue(undefined)
|
||||
vi.mocked(s3Service.putObject).mockResolvedValue(
|
||||
{} as unknown as Awaited<ReturnType<typeof s3Service.putObject>>,
|
||||
)
|
||||
|
||||
const result = await transcodeService.renderPdfPages({
|
||||
assetKey: 'projects/p1/doc.pdf',
|
||||
assetId: 'asset-1',
|
||||
start: 1,
|
||||
end: 1,
|
||||
})
|
||||
|
||||
expect(result.length).toBe(1)
|
||||
expect(result[0].page).toBe(1)
|
||||
expect(result[0].key).toContain('files/asset-1/pdf_pages/doc-page-1-')
|
||||
expect(s3Service.putObject).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
describe('overlayAnnotationsOnBuffer Pixel Tests', () => {
|
||||
it('should draw annotation overlay on a 100x200 image and mutate pixels', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const actualSharp = (await vi.importActual('sharp')) as any
|
||||
const realSharp = (actualSharp.default || actualSharp) as typeof sharp
|
||||
|
||||
const inputBuffer = await realSharp({
|
||||
create: {
|
||||
width: 100,
|
||||
height: 200,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer()
|
||||
|
||||
const annotations: PrismaJson.AnnotationList = [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0.2, 0.2],
|
||||
[0.8, 0.8],
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
const sharpSpy = vi
|
||||
.mocked(sharp)
|
||||
.mockImplementation((input: unknown, options?: unknown) =>
|
||||
realSharp(input as Parameters<typeof sharp>[0], options as Parameters<typeof sharp>[1]),
|
||||
)
|
||||
|
||||
const outputBuffer = await transcodeService.overlayAnnotationsOnBuffer(
|
||||
inputBuffer,
|
||||
annotations,
|
||||
)
|
||||
sharpSpy.mockRestore()
|
||||
|
||||
const { data, info } = await realSharp(outputBuffer)
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true })
|
||||
|
||||
expect(info.width).toBe(100)
|
||||
expect(info.height).toBe(200)
|
||||
|
||||
const pixelX = 50
|
||||
const pixelY = 40
|
||||
const offset = (pixelY * info.width + pixelX) * info.channels
|
||||
|
||||
const r = data[offset]
|
||||
const g = data[offset + 1]
|
||||
const b = data[offset + 2]
|
||||
|
||||
expect(r).toBeGreaterThan(150)
|
||||
expect(g).toBeLessThan(120)
|
||||
expect(b).toBeLessThan(120)
|
||||
})
|
||||
|
||||
it('should draw annotation overlay on a 200x100 image and mutate pixels', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const actualSharp = (await vi.importActual('sharp')) as any
|
||||
const realSharp = (actualSharp.default || actualSharp) as typeof sharp
|
||||
|
||||
const inputBuffer = await realSharp({
|
||||
create: {
|
||||
width: 200,
|
||||
height: 100,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toBuffer()
|
||||
|
||||
const annotations: PrismaJson.AnnotationList = [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#00ff00',
|
||||
points: [
|
||||
[0.1, 0.1],
|
||||
[0.9, 0.9],
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
const sharpSpy = vi
|
||||
.mocked(sharp)
|
||||
.mockImplementation((input: unknown, options?: unknown) =>
|
||||
realSharp(input as Parameters<typeof sharp>[0], options as Parameters<typeof sharp>[1]),
|
||||
)
|
||||
|
||||
const outputBuffer = await transcodeService.overlayAnnotationsOnBuffer(
|
||||
inputBuffer,
|
||||
annotations,
|
||||
)
|
||||
sharpSpy.mockRestore()
|
||||
|
||||
const { data, info } = await realSharp(outputBuffer)
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true })
|
||||
|
||||
expect(info.width).toBe(200)
|
||||
expect(info.height).toBe(100)
|
||||
|
||||
const pixelX = 100
|
||||
const pixelY = 10
|
||||
const offset = (pixelY * info.width + pixelX) * info.channels
|
||||
|
||||
const r = data[offset]
|
||||
const g = data[offset + 1]
|
||||
const b = data[offset + 2]
|
||||
|
||||
expect(r).toBeLessThan(100)
|
||||
expect(g).toBeGreaterThan(200)
|
||||
expect(b).toBeLessThan(100)
|
||||
})
|
||||
|
||||
it('should generate PDF from text file including CJK characters', async () => {
|
||||
const txtFile = path.join(tempDir, 'test.txt')
|
||||
const pdfFile = path.join(tempDir, 'output.pdf')
|
||||
fs.writeFileSync(txtFile, 'Hello World\n你好世界\nこんにちは世界\n안녕하세요世界')
|
||||
|
||||
await transcodeService.generatePdfFromText(txtFile, pdfFile)
|
||||
|
||||
expect(fs.existsSync(pdfFile)).toBe(true)
|
||||
const stat = fs.statSync(pdfFile)
|
||||
expect(stat.size).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should generate PDF from CSV file including CJK characters', async () => {
|
||||
const csvFile = path.join(tempDir, 'test.csv')
|
||||
const pdfFile = path.join(tempDir, 'output.pdf')
|
||||
fs.writeFileSync(
|
||||
csvFile,
|
||||
'Name,Age,Country\nHello World,25,USA\n你好世界,30,China\nこんにちは,28,Japan\n안녕하세요,22,Korea',
|
||||
)
|
||||
|
||||
await transcodeService.generatePdfFromCsv(csvFile, pdfFile)
|
||||
|
||||
expect(fs.existsSync(pdfFile)).toBe(true)
|
||||
const stat = fs.statSync(pdfFile)
|
||||
expect(stat.size).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { stemFromKey } from '@shumai/core/src/utils/filename'
|
||||
import { prisma, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
import '@shumai/db/src/prisma-json-types'
|
||||
import { execFile } from 'child_process'
|
||||
import { parse } from 'csv-parse/sync'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import PDFDocument from 'pdfkit'
|
||||
import sharp from 'sharp'
|
||||
import { ulid } from 'ulid'
|
||||
import { promisify } from 'util'
|
||||
@@ -12,6 +15,77 @@ import { dataFormatNames } from './dataFormatNames'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
export interface CjkFontConfig {
|
||||
fontPath: string
|
||||
fontName?: string
|
||||
}
|
||||
|
||||
export function findCjkFontPath(): CjkFontConfig | undefined {
|
||||
const candidates: { fontPath: string; fontName?: string }[] = [
|
||||
// Custom env override
|
||||
...(process.env.PDF_CJK_FONT_PATH
|
||||
? [
|
||||
{
|
||||
fontPath: process.env.PDF_CJK_FONT_PATH,
|
||||
fontName: process.env.PDF_CJK_FONT_NAME,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
// macOS
|
||||
{ fontPath: '/System/Library/Fonts/Supplemental/Arial Unicode.ttf' },
|
||||
{ fontPath: '/Library/Fonts/Arial Unicode.ttf' },
|
||||
{ fontPath: '/System/Library/Fonts/PingFang.ttc', fontName: 'PingFangSC-Regular' },
|
||||
{ fontPath: '/System/Library/Fonts/STHeiti Light.ttc', fontName: 'STHeitiSC-Light' },
|
||||
// Linux / Ubuntu / Debian Noto & WenQuanYi CJK
|
||||
{
|
||||
fontPath: '/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc',
|
||||
fontName: 'NotoSansCJKsc-Regular',
|
||||
},
|
||||
{
|
||||
fontPath: '/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc',
|
||||
fontName: 'NotoSansCJKsc-Regular',
|
||||
},
|
||||
{ fontPath: '/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc', fontName: 'WenQuanYiZenHei' },
|
||||
{ fontPath: '/usr/share/fonts/truetype/arphic/ukai.ttc', fontName: 'AR-PL-UKai-CN' },
|
||||
{
|
||||
fontPath: '/usr/share/fonts/noto/NotoSansCJK-Regular.ttc',
|
||||
fontName: 'Noto Sans CJK SC',
|
||||
},
|
||||
// Lightweight Linux / Docker Droid Fallbacks
|
||||
{ fontPath: '/usr/share/fonts/truetype/droid/DroidSansFallbackFull.ttf' },
|
||||
{ fontPath: '/usr/share/fonts/truetype/droid/DroidSansFallback.ttf' },
|
||||
{ fontPath: '/usr/share/fonts/google-droid/DroidSansFallback.ttf' },
|
||||
// Windows CJK Fonts
|
||||
{ fontPath: 'C:\\Windows\\Fonts\\msyh.ttc', fontName: 'MicrosoftYaHei' },
|
||||
{ fontPath: 'C:\\Windows\\Fonts\\simsun.ttc', fontName: 'SimSun' },
|
||||
{ fontPath: 'C:\\Windows\\Fonts\\simhei.ttf' },
|
||||
]
|
||||
for (const item of candidates) {
|
||||
if (fs.existsSync(item.fontPath)) {
|
||||
return item
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function parseCsvContent(content: string): string[][] {
|
||||
try {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
return parse(content, {
|
||||
skip_empty_lines: true,
|
||||
relax_column_count: true,
|
||||
relax_quotes: true,
|
||||
trim: true,
|
||||
delimiter_auto: true,
|
||||
})
|
||||
} catch {
|
||||
return content
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => line.split(',').map((cell) => cell.trim()))
|
||||
}
|
||||
}
|
||||
|
||||
export interface MediaMetadata {
|
||||
originalWidth: number
|
||||
originalHeight: number
|
||||
@@ -287,6 +361,271 @@ export class TranscodeService {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
|
||||
async generatePdfSprite(
|
||||
inputFile: string,
|
||||
outputSprite: string,
|
||||
outputPoster: string,
|
||||
): Promise<{ pageCount: number; originalWidth: number; originalHeight: number }> {
|
||||
const tmpDir = this.createTempDir('pdf-sprite-')
|
||||
try {
|
||||
const pagePrefix = path.join(tmpDir, 'page')
|
||||
try {
|
||||
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix])
|
||||
} catch (err) {
|
||||
const errCode = (err as Record<string, unknown>)?.code
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
const lower = msg.toLowerCase()
|
||||
if (
|
||||
errCode === 'ENOENT' ||
|
||||
lower.includes('enoent') ||
|
||||
(lower.includes('not found') && lower.includes('pdftoppm'))
|
||||
) {
|
||||
throw new Error(
|
||||
`pdftoppm executable not found in $PATH. Please install poppler-utils / poppler. (${msg})`,
|
||||
{ cause: err },
|
||||
)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png'))
|
||||
if (files.length === 0) {
|
||||
throw new Error('pdftoppm produced no image outputs')
|
||||
}
|
||||
|
||||
files.sort((a, b) => {
|
||||
const numA = parseInt(a.replace(/[^0-9]/g, ''), 10)
|
||||
const numB = parseInt(b.replace(/[^0-9]/g, ''), 10)
|
||||
return numA - numB
|
||||
})
|
||||
|
||||
const extractedCount = files.length
|
||||
const firstPagePath = path.join(tmpDir, files[0])
|
||||
const firstMeta = await sharp(firstPagePath, { limitInputPixels: false }).metadata()
|
||||
const originalWidth = firstMeta.width || 800
|
||||
const originalHeight = firstMeta.height || 1000
|
||||
|
||||
for (let f = 1; f <= 100; f++) {
|
||||
const pageIdx = Math.min(extractedCount - 1, Math.floor(((f - 1) * extractedCount) / 100))
|
||||
const srcPath = path.join(tmpDir, files[pageIdx])
|
||||
const destPath = path.join(tmpDir, `frame_${f}.png`)
|
||||
if (srcPath !== destPath) {
|
||||
fs.copyFileSync(srcPath, destPath)
|
||||
}
|
||||
}
|
||||
|
||||
const spriteArgs = [
|
||||
'-i',
|
||||
path.join(tmpDir, 'frame_%d.png'),
|
||||
'-filter_complex',
|
||||
'scale=w=480:h=-2,tile=10x10',
|
||||
'-frames:v',
|
||||
'1',
|
||||
'-c:v',
|
||||
'libwebp',
|
||||
'-q:v',
|
||||
'75',
|
||||
outputSprite,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs])
|
||||
|
||||
await sharp(firstPagePath, { limitInputPixels: false })
|
||||
.resize(480, null, { withoutEnlargement: true, fit: 'inside' })
|
||||
.webp({ quality: 75 })
|
||||
.toFile(outputPoster)
|
||||
|
||||
let totalPages = extractedCount
|
||||
try {
|
||||
const { stdout } = await execFileAsync('pdfinfo', [inputFile])
|
||||
const match = stdout.match(/Pages:\s+(\d+)/)
|
||||
if (match) {
|
||||
totalPages = parseInt(match[1], 10)
|
||||
}
|
||||
} catch {
|
||||
// Fallback to extracted count if pdfinfo is unavailable
|
||||
}
|
||||
|
||||
return {
|
||||
pageCount: totalPages,
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
}
|
||||
} finally {
|
||||
this.removeDir(tmpDir)
|
||||
}
|
||||
}
|
||||
|
||||
async getPdfInfo(inputFile: string): Promise<MediaMetadata> {
|
||||
const tmpDir = this.createTempDir('pdf-info-')
|
||||
let originalWidth = 800
|
||||
let originalHeight = 1000
|
||||
let pageCount = 1
|
||||
|
||||
try {
|
||||
const pagePrefix = path.join(tmpDir, 'page')
|
||||
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '1', inputFile, pagePrefix])
|
||||
const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png'))
|
||||
if (files.length > 0) {
|
||||
const meta = await sharp(path.join(tmpDir, files[0]), {
|
||||
limitInputPixels: false,
|
||||
}).metadata()
|
||||
originalWidth = meta.width || 800
|
||||
originalHeight = meta.height || 1000
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('pdftoppm not available for PDF dimensions extraction:', err)
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout } = await execFileAsync('pdfinfo', [inputFile])
|
||||
const match = stdout.match(/Pages:\s+(\d+)/)
|
||||
if (match) {
|
||||
pageCount = parseInt(match[1], 10)
|
||||
}
|
||||
} catch {
|
||||
// Fallback
|
||||
} finally {
|
||||
this.removeDir(tmpDir)
|
||||
}
|
||||
|
||||
return {
|
||||
originalWidth,
|
||||
originalHeight,
|
||||
duration: 0,
|
||||
bitRate: 0,
|
||||
frameRate: 0,
|
||||
totalFrames: pageCount,
|
||||
hasAudio: false,
|
||||
mimeType: 'application/pdf',
|
||||
}
|
||||
}
|
||||
|
||||
async generatePdfFromText(inputFile: string, outputFile: string): Promise<void> {
|
||||
const content = fs.readFileSync(inputFile, 'utf-8')
|
||||
const containsCjk = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9fff\uac00-\ud7af]/.test(
|
||||
content,
|
||||
)
|
||||
const cjkFont = containsCjk ? findCjkFontPath() : undefined
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const doc = new PDFDocument({
|
||||
size: 'A4',
|
||||
margin: 40,
|
||||
bufferPages: true,
|
||||
})
|
||||
const writeStream = fs.createWriteStream(outputFile)
|
||||
doc.pipe(writeStream)
|
||||
|
||||
if (cjkFont) {
|
||||
if (cjkFont.fontName !== undefined) {
|
||||
doc.font(cjkFont.fontPath, cjkFont.fontName)
|
||||
} else {
|
||||
doc.font(cjkFont.fontPath)
|
||||
}
|
||||
} else {
|
||||
doc.font('Helvetica')
|
||||
}
|
||||
doc.fontSize(10)
|
||||
|
||||
doc.text(content, {
|
||||
lineGap: 3,
|
||||
paragraphGap: 4,
|
||||
})
|
||||
|
||||
doc.end()
|
||||
|
||||
writeStream.on('finish', () => resolve())
|
||||
writeStream.on('error', (err) => reject(err))
|
||||
})
|
||||
}
|
||||
|
||||
async generatePdfFromCsv(inputFile: string, outputFile: string): Promise<void> {
|
||||
const content = fs.readFileSync(inputFile, 'utf-8')
|
||||
const rows = parseCsvContent(content)
|
||||
const containsCjk = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9fff\uac00-\ud7af]/.test(
|
||||
content,
|
||||
)
|
||||
const cjkFont = containsCjk ? findCjkFontPath() : undefined
|
||||
|
||||
if (rows.length === 0) {
|
||||
rows.push(['(Empty CSV)'])
|
||||
}
|
||||
|
||||
const maxCols = Math.max(...rows.map((r) => r.length))
|
||||
const isLandscape = maxCols > 5
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const doc = new PDFDocument({
|
||||
size: 'A4',
|
||||
layout: isLandscape ? 'landscape' : 'portrait',
|
||||
margin: 30,
|
||||
bufferPages: true,
|
||||
})
|
||||
const writeStream = fs.createWriteStream(outputFile)
|
||||
doc.pipe(writeStream)
|
||||
|
||||
const setFont = () => {
|
||||
if (cjkFont) {
|
||||
if (cjkFont.fontName !== undefined) {
|
||||
doc.font(cjkFont.fontPath, cjkFont.fontName)
|
||||
} else {
|
||||
doc.font(cjkFont.fontPath)
|
||||
}
|
||||
} else {
|
||||
doc.font('Helvetica')
|
||||
}
|
||||
doc.fontSize(9)
|
||||
}
|
||||
|
||||
setFont()
|
||||
|
||||
const pageWidth = doc.page.width - doc.page.margins.left - doc.page.margins.right
|
||||
const colWidth = Math.max(40, pageWidth / maxCols)
|
||||
|
||||
const startX = doc.page.margins.left
|
||||
let startY = doc.page.margins.top
|
||||
const rowHeight = 20
|
||||
|
||||
const drawRow = (row: string[], isHeader: boolean, y: number) => {
|
||||
if (isHeader) {
|
||||
doc.rect(startX, y, colWidth * maxCols, rowHeight).fill('#e0e0e0')
|
||||
doc.fillColor('#000000')
|
||||
}
|
||||
for (let col = 0; col < maxCols; col++) {
|
||||
const text = row[col] || ''
|
||||
const x = startX + col * colWidth
|
||||
doc.rect(x, y, colWidth, rowHeight).stroke('#cccccc')
|
||||
doc.fillColor('#000000').text(text, x + 4, y + 5, {
|
||||
width: colWidth - 8,
|
||||
height: rowHeight - 6,
|
||||
ellipsis: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const headerRow = rows[0]
|
||||
drawRow(headerRow, true, startY)
|
||||
startY += rowHeight
|
||||
|
||||
for (let r = 1; r < rows.length; r++) {
|
||||
if (startY + rowHeight > doc.page.height - doc.page.margins.bottom) {
|
||||
doc.addPage()
|
||||
setFont()
|
||||
startY = doc.page.margins.top
|
||||
drawRow(headerRow, true, startY)
|
||||
startY += rowHeight
|
||||
}
|
||||
drawRow(rows[r], false, startY)
|
||||
startY += rowHeight
|
||||
}
|
||||
|
||||
doc.end()
|
||||
|
||||
writeStream.on('finish', () => resolve())
|
||||
writeStream.on('error', (err) => reject(err))
|
||||
})
|
||||
}
|
||||
|
||||
async extractAudio(inputFile: string, outputFile: string, bitrate: string): Promise<void> {
|
||||
const args = ['-i', inputFile, '-vn', '-acodec', 'libmp3lame', '-b:a', bitrate, outputFile]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
@@ -403,7 +742,7 @@ export class TranscodeService {
|
||||
data: {
|
||||
assetId,
|
||||
projectId,
|
||||
type: WorkflowTaskType.transcode,
|
||||
type: WorkflowTaskType.transcode_video,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId,
|
||||
@@ -418,7 +757,7 @@ export class TranscodeService {
|
||||
data: {
|
||||
assetId,
|
||||
projectId,
|
||||
type: WorkflowTaskType.transcode,
|
||||
type: WorkflowTaskType.transcode_image,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId,
|
||||
@@ -503,18 +842,9 @@ export class TranscodeService {
|
||||
params.annotations &&
|
||||
params.annotations.length > 0
|
||||
) {
|
||||
const meta = await sharp(localShotPath, { limitInputPixels: false }).metadata()
|
||||
const width = meta.width || 1280
|
||||
const height = meta.height || 720
|
||||
|
||||
const svgStr = renderAnnotationsToSvg(width, height, params.annotations)
|
||||
const tempCompositedPath = path.join(tmpDir, `composite-${outName}`)
|
||||
|
||||
await sharp(localShotPath, { limitInputPixels: false })
|
||||
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
|
||||
.toFile(tempCompositedPath)
|
||||
|
||||
fs.renameSync(tempCompositedPath, localShotPath)
|
||||
const shotBuffer = fs.readFileSync(localShotPath)
|
||||
const composited = await this.overlayAnnotationsOnBuffer(shotBuffer, params.annotations)
|
||||
fs.writeFileSync(localShotPath, composited)
|
||||
}
|
||||
|
||||
// 6. Upload to S3
|
||||
@@ -531,41 +861,153 @@ export class TranscodeService {
|
||||
}
|
||||
}
|
||||
|
||||
async renderPdfPages(params: {
|
||||
assetKey: string
|
||||
assetId: string
|
||||
start: number
|
||||
end: number
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
}): Promise<Array<{ key: string; page: number }>> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const tmpDir = this.createTempDir('pdf-pages-')
|
||||
const pdfPath = path.join(tmpDir, path.basename(params.assetKey))
|
||||
|
||||
try {
|
||||
// 1. Download PDF file
|
||||
await s3Service.downloadToFile(bucket, params.assetKey, pdfPath)
|
||||
|
||||
// 2. Render pages via pdftoppm (-png -f start -l end)
|
||||
const pagePrefix = path.join(tmpDir, 'page')
|
||||
try {
|
||||
await execFileAsync('pdftoppm', [
|
||||
'-png',
|
||||
'-f',
|
||||
params.start.toString(),
|
||||
'-l',
|
||||
params.end.toString(),
|
||||
pdfPath,
|
||||
pagePrefix,
|
||||
])
|
||||
} catch (err) {
|
||||
const errCode = (err as Record<string, unknown>)?.code
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
const lower = msg.toLowerCase()
|
||||
if (
|
||||
errCode === 'ENOENT' ||
|
||||
lower.includes('enoent') ||
|
||||
(lower.includes('not found') && lower.includes('pdftoppm'))
|
||||
) {
|
||||
throw new Error(
|
||||
`pdftoppm executable not found in $PATH. Please install poppler-utils / poppler. (${msg})`,
|
||||
{ cause: err },
|
||||
)
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png'))
|
||||
if (files.length === 0) {
|
||||
throw new Error('pdftoppm produced no image outputs')
|
||||
}
|
||||
|
||||
files.sort((a, b) => {
|
||||
const numA = parseInt(a.replace(/[^0-9]/g, ''), 10)
|
||||
const numB = parseInt(b.replace(/[^0-9]/g, ''), 10)
|
||||
return numA - numB
|
||||
})
|
||||
|
||||
const results: Array<{ key: string; page: number }> = []
|
||||
const stem = stemFromKey(params.assetKey)
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const pageNum = params.start + i
|
||||
const localPngPath = path.join(tmpDir, files[i])
|
||||
const webpName = `${stem}-page-${pageNum}-${ulid()}.webp`
|
||||
|
||||
let webpBuffer = await sharp(localPngPath, { limitInputPixels: false })
|
||||
.resize(1920, 1080, { fit: 'inside', withoutEnlargement: false })
|
||||
.webp({ quality: 85 })
|
||||
.toBuffer()
|
||||
|
||||
// Overlay annotations if comment page matches pageNum
|
||||
const commentPage =
|
||||
params.commentTimestamp !== undefined && params.commentTimestamp !== null
|
||||
? Math.round(params.commentTimestamp)
|
||||
: null
|
||||
|
||||
if (
|
||||
commentPage !== null &&
|
||||
pageNum === commentPage &&
|
||||
params.annotations &&
|
||||
params.annotations.length > 0
|
||||
) {
|
||||
webpBuffer = await this.overlayAnnotationsOnBuffer(webpBuffer, params.annotations)
|
||||
}
|
||||
|
||||
// Upload to S3 directly from buffer
|
||||
const s3Key = `files/${params.assetId}/pdf_pages/${webpName}`
|
||||
await s3Service.putObject(bucket, s3Key, webpBuffer, webpBuffer.length, 'image/webp')
|
||||
|
||||
results.push({ key: s3Key, page: pageNum })
|
||||
}
|
||||
|
||||
return results
|
||||
} finally {
|
||||
this.removeDir(tmpDir)
|
||||
}
|
||||
}
|
||||
|
||||
async overlayAnnotationsOnBuffer(
|
||||
imageBuffer: Buffer,
|
||||
annotations: PrismaJson.AnnotationList,
|
||||
): Promise<Buffer> {
|
||||
if (!annotations || annotations.length === 0) {
|
||||
return imageBuffer
|
||||
}
|
||||
|
||||
const meta = await sharp(imageBuffer, { limitInputPixels: false }).metadata()
|
||||
const width = meta.width || 1920
|
||||
const height = meta.height || 1080
|
||||
|
||||
const svgStr = renderAnnotationsToSvg(width, height, annotations)
|
||||
return await sharp(imageBuffer, { limitInputPixels: false })
|
||||
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
|
||||
.resize(16383, 16383, { fit: 'inside', withoutEnlargement: true })
|
||||
.webp({ quality: 85 })
|
||||
.toBuffer()
|
||||
}
|
||||
|
||||
async overlayAnnotations(params: {
|
||||
assetKey: string
|
||||
assetId: string
|
||||
annotations: PrismaJson.AnnotationList
|
||||
}): Promise<string> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const outName = `annotation-${ulid()}.webp`
|
||||
const tmpDir = this.createTempDir('annotation-')
|
||||
const imgPath = path.join(tmpDir, path.basename(params.assetKey))
|
||||
|
||||
try {
|
||||
// 1. Download image
|
||||
await s3Service.downloadToFile(bucket, params.assetKey, imgPath)
|
||||
const inputBuffer = fs.readFileSync(imgPath)
|
||||
|
||||
// 2. Read dimensions
|
||||
const meta = await sharp(imgPath, { limitInputPixels: false }).metadata()
|
||||
const width = meta.width || 1920
|
||||
const height = meta.height || 1080
|
||||
// 2. Overlay annotations in memory using helper method
|
||||
const compositedBuffer = await this.overlayAnnotationsOnBuffer(
|
||||
inputBuffer,
|
||||
params.annotations,
|
||||
)
|
||||
|
||||
// 3. Render SVG
|
||||
const svgStr = renderAnnotationsToSvg(width, height, params.annotations)
|
||||
|
||||
// 4. Composite
|
||||
const outName = `annotation-${ulid()}.webp`
|
||||
const localOutPath = path.join(tmpDir, outName)
|
||||
|
||||
await sharp(imgPath, { limitInputPixels: false })
|
||||
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
|
||||
.resize(16383, 16383, { fit: 'inside', withoutEnlargement: true })
|
||||
.webp({ quality: 90 })
|
||||
.toFile(localOutPath)
|
||||
|
||||
// 5. Upload to S3
|
||||
// 3. Upload to S3
|
||||
const s3Key = `files/${params.assetId}/annotations/${outName}`
|
||||
const fileBuffer = fs.readFileSync(localOutPath)
|
||||
await s3Service.putObject(bucket, s3Key, fileBuffer, fileBuffer.length, 'image/webp')
|
||||
await s3Service.putObject(
|
||||
bucket,
|
||||
s3Key,
|
||||
compositedBuffer,
|
||||
compositedBuffer.length,
|
||||
'image/webp',
|
||||
)
|
||||
|
||||
return s3Key
|
||||
} finally {
|
||||
|
||||
@@ -182,7 +182,7 @@ describe('UploadService', () => {
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.uploaded)
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
|
||||
})
|
||||
expect(workflowTask).toBeDefined()
|
||||
expect(workflowTask?.payload).toEqual({
|
||||
@@ -219,7 +219,7 @@ describe('UploadService', () => {
|
||||
await uploadService.confirmFileUpload(userId, task.id, { fileId: asset.id })
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_image },
|
||||
})
|
||||
expect(workflowTask).toBeDefined()
|
||||
expect(workflowTask?.payload).toEqual({
|
||||
@@ -236,7 +236,7 @@ describe('UploadService', () => {
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'file.txt',
|
||||
name: 'file.json',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
@@ -247,7 +247,7 @@ describe('UploadService', () => {
|
||||
create: { key: 'test-key' },
|
||||
},
|
||||
},
|
||||
mediaType: 'text/plain',
|
||||
mediaType: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
@@ -257,7 +257,7 @@ describe('UploadService', () => {
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
|
||||
})
|
||||
expect(workflowTask).toBeNull()
|
||||
})
|
||||
@@ -290,7 +290,7 @@ describe('UploadService', () => {
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.uploaded)
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
|
||||
})
|
||||
expect(workflowTask).toBeDefined()
|
||||
})
|
||||
@@ -323,7 +323,7 @@ describe('UploadService', () => {
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.uploaded)
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode },
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_video },
|
||||
})
|
||||
expect(workflowTask).toBeDefined()
|
||||
})
|
||||
|
||||
@@ -18,10 +18,11 @@ import {
|
||||
PresignedUrl,
|
||||
TaskInfo,
|
||||
} from '@shumai/dtos'
|
||||
import { ImageTranscoder, VideoTranscoder } from '@shumai/transcode'
|
||||
import { ImageTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transcode'
|
||||
import { generateKeyBetween } from 'jittered-fractional-indexing'
|
||||
import { ulid } from 'ulid'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
|
||||
export class UploadService {
|
||||
constructor(private readonly prismaClient: typeof prisma = prisma) {}
|
||||
@@ -300,9 +301,14 @@ export class UploadService {
|
||||
},
|
||||
})
|
||||
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
const isImage = asset.mediaType?.startsWith('image/')
|
||||
const isAudio = asset.mediaType?.startsWith('audio/')
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
|
||||
const isVideo = proxyType === 'video'
|
||||
const isImage = proxyType === 'image'
|
||||
const isAudio = proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
|
||||
if (autofillAgent && (isVideo || isImage)) {
|
||||
await tx.workflowTask.create({
|
||||
@@ -320,7 +326,7 @@ export class UploadService {
|
||||
})
|
||||
}
|
||||
|
||||
if (!isVideo && !isImage && !isAudio) {
|
||||
if (!proxyType) {
|
||||
await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
@@ -341,6 +347,12 @@ export class UploadService {
|
||||
} else if (isImage) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new ImageTranscoder(tx as any, asset.id, team.id, projectId).withThumbnail().submit()
|
||||
} else if (isPdf) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new PdfTranscoder(tx as any, asset.id, team.id, projectId)
|
||||
.withSprite()
|
||||
.withPoster()
|
||||
.submit()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { detectSupportedMimeType } from './mime'
|
||||
import { detectSupportedMimeType, getProxyType } from './mime'
|
||||
|
||||
describe('detectSupportedMimeType', () => {
|
||||
it('should detect JPEG', () => {
|
||||
@@ -39,3 +39,36 @@ describe('detectSupportedMimeType', () => {
|
||||
expect(detectSupportedMimeType(buffer)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getProxyType', () => {
|
||||
it('should detect image proxyType', () => {
|
||||
expect(getProxyType('image/png', 'test.png')).toBe('image')
|
||||
expect(getProxyType('image/jpeg', 'photo.jpg')).toBe('image')
|
||||
})
|
||||
|
||||
it('should detect video proxyType', () => {
|
||||
expect(getProxyType('video/mp4', 'clip.mp4')).toBe('video')
|
||||
})
|
||||
|
||||
it('should detect audio proxyType', () => {
|
||||
expect(getProxyType('audio/mpeg', 'song.mp3')).toBe('audio')
|
||||
})
|
||||
|
||||
it('should detect pdf proxyType only for pdf, csv, txt, and markdown files', () => {
|
||||
expect(getProxyType('application/pdf', 'doc.pdf')).toBe('pdf')
|
||||
expect(getProxyType('text/plain', 'notes.txt')).toBe('pdf')
|
||||
expect(getProxyType('text/csv', 'data.csv')).toBe('pdf')
|
||||
expect(getProxyType('text/markdown', 'README.md')).toBe('pdf')
|
||||
expect(getProxyType('text/x-markdown', 'doc.markdown')).toBe('pdf')
|
||||
expect(getProxyType(null, 'file.txt')).toBe('pdf')
|
||||
expect(getProxyType(null, 'README.md')).toBe('pdf')
|
||||
expect(getProxyType(null, 'doc.markdown')).toBe('pdf')
|
||||
})
|
||||
|
||||
it('should return null for unsupported files including office files', () => {
|
||||
expect(getProxyType('application/zip', 'archive.zip')).toBeNull()
|
||||
expect(getProxyType('application/msword', 'letter.doc')).toBeNull()
|
||||
expect(getProxyType(null, 'sheet.xlsx')).toBeNull()
|
||||
expect(getProxyType(null, 'unknown.bin')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,3 +1,34 @@
|
||||
export type ProxyType = 'image' | 'video' | 'audio' | 'pdf'
|
||||
|
||||
export function getProxyType(
|
||||
mediaType?: string | null,
|
||||
filename?: string | null,
|
||||
): ProxyType | null {
|
||||
const lowerMediaType = mediaType?.toLowerCase() || ''
|
||||
const lowerFilename = filename?.toLowerCase() || ''
|
||||
|
||||
if (lowerMediaType.startsWith('image/')) return 'image'
|
||||
if (lowerMediaType.startsWith('video/')) return 'video'
|
||||
if (lowerMediaType.startsWith('audio/')) return 'audio'
|
||||
|
||||
if (
|
||||
lowerMediaType === 'application/pdf' ||
|
||||
lowerMediaType === 'text/plain' ||
|
||||
lowerMediaType === 'text/csv' ||
|
||||
lowerMediaType === 'text/markdown' ||
|
||||
lowerMediaType === 'text/x-markdown' ||
|
||||
lowerFilename.endsWith('.pdf') ||
|
||||
lowerFilename.endsWith('.txt') ||
|
||||
lowerFilename.endsWith('.csv') ||
|
||||
lowerFilename.endsWith('.md') ||
|
||||
lowerFilename.endsWith('.markdown')
|
||||
) {
|
||||
return 'pdf'
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
|
||||
|
||||
export function detectSupportedMimeType(buffer: Uint8Array): string | null {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/db",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
-- AlterEnum
|
||||
-- This migration adds more than one value to an enum.
|
||||
-- With PostgreSQL versions 11 and earlier, this is not possible
|
||||
-- in a single migration. This can be worked around by creating
|
||||
-- multiple migrations, each migration adding only one value to
|
||||
-- the enum.
|
||||
|
||||
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_video';
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_image';
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_pdf';
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_pdf_pages';
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_screenshot';
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_image_annotation';
|
||||
@@ -0,0 +1,32 @@
|
||||
-- Backfill media.proxyType for legacy assets
|
||||
UPDATE "assets"
|
||||
SET "media" = jsonb_set(
|
||||
COALESCE("media"::jsonb, '{}'::jsonb),
|
||||
'{proxyType}',
|
||||
'"video"'::jsonb
|
||||
)
|
||||
WHERE "media" IS NOT NULL AND "media_type" LIKE 'video/%';
|
||||
|
||||
UPDATE "assets"
|
||||
SET "media" = jsonb_set(
|
||||
COALESCE("media"::jsonb, '{}'::jsonb),
|
||||
'{proxyType}',
|
||||
'"audio"'::jsonb
|
||||
)
|
||||
WHERE "media" IS NOT NULL AND "media_type" LIKE 'audio/%';
|
||||
|
||||
UPDATE "assets"
|
||||
SET "media" = jsonb_set(
|
||||
COALESCE("media"::jsonb, '{}'::jsonb),
|
||||
'{proxyType}',
|
||||
'"image"'::jsonb
|
||||
)
|
||||
WHERE "media" IS NOT NULL AND "media_type" LIKE 'image/%';
|
||||
|
||||
UPDATE "assets"
|
||||
SET "media" = jsonb_set(
|
||||
COALESCE("media"::jsonb, '{}'::jsonb),
|
||||
'{proxyType}',
|
||||
'"pdf"'::jsonb
|
||||
)
|
||||
WHERE "media" IS NOT NULL AND ("media_type" = 'application/pdf' OR "name" ILIKE '%.pdf');
|
||||
@@ -100,6 +100,12 @@ enum TeamMemberScope {
|
||||
|
||||
enum WorkflowTaskType {
|
||||
transcode
|
||||
transcode_video
|
||||
transcode_image
|
||||
transcode_pdf
|
||||
transcode_pdf_pages
|
||||
transcode_screenshot
|
||||
transcode_image_annotation
|
||||
ai_metadata_autofill
|
||||
chat
|
||||
ai_embedding
|
||||
|
||||
+6
@@ -126,6 +126,12 @@ export type TeamMemberScope = (typeof TeamMemberScope)[keyof typeof TeamMemberSc
|
||||
|
||||
export const WorkflowTaskType = {
|
||||
transcode: 'transcode',
|
||||
transcode_video: 'transcode_video',
|
||||
transcode_image: 'transcode_image',
|
||||
transcode_pdf: 'transcode_pdf',
|
||||
transcode_pdf_pages: 'transcode_pdf_pages',
|
||||
transcode_screenshot: 'transcode_screenshot',
|
||||
transcode_image_annotation: 'transcode_image_annotation',
|
||||
ai_metadata_autofill: 'ai_metadata_autofill',
|
||||
chat: 'chat',
|
||||
ai_embedding: 'ai_embedding',
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -30,6 +30,7 @@ declare global {
|
||||
thinkingLevel?: 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh'
|
||||
systemPrompt?: string
|
||||
appendSystemPrompt?: string[]
|
||||
deniedTools?: string[]
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
@@ -123,19 +124,25 @@ declare global {
|
||||
codec: string
|
||||
}
|
||||
|
||||
export interface PdfTranscode {
|
||||
key?: string
|
||||
url?: string
|
||||
}
|
||||
|
||||
export interface MediaInfo {
|
||||
duration: number
|
||||
filesize: number
|
||||
frames: number
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf'
|
||||
imageTranscodes: ImageTranscode[]
|
||||
videoTranscodes: VideoTranscode[]
|
||||
videoPreview: VideoTranscode
|
||||
pdfTranscode?: PdfTranscode
|
||||
sprite?: SpriteInfo
|
||||
poster?: PosterInfo
|
||||
thumbnail?: ImageTranscode
|
||||
finishedAt: string
|
||||
metadata: Metadata | null
|
||||
mimeType: string
|
||||
original: OriginalInfo | null
|
||||
}
|
||||
|
||||
@@ -254,6 +261,13 @@ declare global {
|
||||
annotations?: AnnotationList | null
|
||||
}
|
||||
|
||||
export interface PdfPagesSpec {
|
||||
start: number
|
||||
end: number
|
||||
commentTimestamp?: number | null
|
||||
annotations?: AnnotationList | null
|
||||
}
|
||||
|
||||
export interface WorkflowTaskPayload {
|
||||
projectId: string
|
||||
transcode?: TaskSpec
|
||||
@@ -264,6 +278,7 @@ declare global {
|
||||
agentToolCall?: AgentToolCallPayload
|
||||
screenshot?: ScreenshotSpec
|
||||
imageAnnotation?: ImageAnnotationSpec
|
||||
pdfPages?: PdfPagesSpec
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
@@ -293,6 +308,7 @@ declare global {
|
||||
attachedFiles?: string[]
|
||||
assetIds?: string[]
|
||||
isNewChat?: boolean
|
||||
hasAssetChanged?: boolean
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
@@ -235,7 +235,7 @@ export class PrismaTestingHelper<
|
||||
})
|
||||
throw internalRollbackErrorSymbol
|
||||
},
|
||||
{ ...opts, timeout: 30000 },
|
||||
{ timeout: 60000, ...opts },
|
||||
)
|
||||
.catch((error) => {
|
||||
if (error !== internalRollbackErrorSymbol) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -30,6 +30,7 @@ export const agentInfoSchema = z.object({
|
||||
systemPrompt: z.string().optional(),
|
||||
soul: z.string().optional(),
|
||||
skills: z.array(agentSkillSchema).optional(),
|
||||
deniedTools: z.array(z.string()).optional(),
|
||||
})
|
||||
export type AgentInfo = z.infer<typeof agentInfoSchema>
|
||||
|
||||
@@ -44,6 +45,7 @@ const baseAgentRequest = z.object({
|
||||
systemPrompt: z.string().optional(),
|
||||
soul: z.string().optional(),
|
||||
skills: z.array(z.string()).optional(),
|
||||
deniedTools: z.array(z.string()).optional(),
|
||||
})
|
||||
|
||||
export const createAgentRequestSchema = baseAgentRequest.superRefine((data, ctx) => {
|
||||
|
||||
@@ -8,12 +8,13 @@ export const assetInfoPaginatedListSchema = z.object({
|
||||
export type AssetInfoPaginatedList = z.infer<typeof assetInfoPaginatedListSchema>
|
||||
|
||||
export const previewInfoSchema = z.object({
|
||||
mediaType: z.string().nullable(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
thumbnailUrl: z.string().optional(),
|
||||
originalHeight: z.number().optional(),
|
||||
originalWidth: z.number().optional(),
|
||||
spriteUrl: z.string().optional(),
|
||||
duration: z.number().optional(),
|
||||
pageCount: z.number().optional(),
|
||||
})
|
||||
export type PreviewInfo = z.infer<typeof previewInfoSchema>
|
||||
|
||||
@@ -61,7 +62,7 @@ export const assetInfoSchema = z.object({
|
||||
type: z.string(),
|
||||
targetType: z.string().optional().nullable(),
|
||||
status: z.string(),
|
||||
mediaType: z.string().nullable(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
latestChildren: z.array(childPreviewSchema).optional(),
|
||||
preview: previewInfoSchema.nullable().optional(),
|
||||
createdAt: z.string(),
|
||||
@@ -104,7 +105,8 @@ export const assetInfoSchema = z.object({
|
||||
)
|
||||
.optional(),
|
||||
videoPreview: z.object({ url: z.string(), key: z.string().optional() }).optional(),
|
||||
mimeType: z.string().optional(),
|
||||
pdfTranscode: z.object({ url: z.string(), key: z.string().optional() }).optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
metadata: mediaMetadataSchema.optional(),
|
||||
})
|
||||
.optional(),
|
||||
@@ -186,7 +188,7 @@ export const attachmentInfoSchema = z.object({
|
||||
id: z.string(),
|
||||
assetId: z.string(),
|
||||
url: z.string(),
|
||||
mediaType: z.string().nullable(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
})
|
||||
export type AttachmentInfo = z.infer<typeof attachmentInfoSchema>
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ export type EntityInfo = z.infer<typeof entityInfoSchema>
|
||||
|
||||
export const assetPreviewSchema = z.object({
|
||||
id: z.string().optional(),
|
||||
mediaType: z.string().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
name: z.string().optional(),
|
||||
preview: z.string().optional(),
|
||||
thumbnailUrl: z.string().optional(),
|
||||
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 68 B |
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "@shumai/e2e",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/workflow-core": "workspace:*",
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/transcode": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@google/genai": "1.52.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"ignoreDeprecations": "6.0"
|
||||
},
|
||||
"include": ["workflow/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [tsconfigPaths()],
|
||||
test: {
|
||||
globalSetup: './packages/e2e/workflow/test-global-setup.ts',
|
||||
globals: false,
|
||||
pool: 'forks',
|
||||
fileParallelism: false,
|
||||
sequence: {
|
||||
concurrent: false,
|
||||
},
|
||||
include: ['packages/e2e/workflow/**/*.test.ts'],
|
||||
server: {
|
||||
deps: {
|
||||
inline: ['zod', '@shumai/workflow-core', '@temporalio/activity'],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,336 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService, TaskQueueAgent, TaskQueueTranscode } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { AgentHarness, type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts')
|
||||
const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts')
|
||||
|
||||
describe.each(['local', 'temporal'] as const)(
|
||||
'Workflow E2E - agentAutofillMedia (executor: %s)',
|
||||
(mode) => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let agentWorkerPromise: Promise<void> | null = null
|
||||
let transcodeWorkerPromise: Promise<void> | null = null
|
||||
|
||||
beforeAll(async () => {
|
||||
// Set bucket environment
|
||||
process.env.S3_BUCKET = 'shumai-e2e-test-bucket'
|
||||
|
||||
// Configure the workflow service executor dynamically
|
||||
workflowService.setExecutorType(mode)
|
||||
|
||||
// Initialize registries
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
|
||||
// Spy on the harness prompt method to intercept the LLM call while running real activities & tools
|
||||
vi.spyOn(AgentHarness.prototype, 'prompt').mockImplementation(async function (
|
||||
this: AgentHarness,
|
||||
) {
|
||||
// Find the real autofill_metadata tool configured on the harness
|
||||
const tools = this.getTools()
|
||||
const autofillTool = tools.find((t: AgentTool) => t.name === 'autofill_metadata')
|
||||
|
||||
if (autofillTool) {
|
||||
// Execute the real tool callback to perform the actual DB updates
|
||||
await autofillTool.execute('call-123', {
|
||||
title: 'E2E Title Extracted',
|
||||
confidence: 0.95,
|
||||
completed: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Return a successful assistant response mock
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any -- Mock assistant response structure */
|
||||
return {
|
||||
content: [{ type: 'text', text: 'Success' }],
|
||||
usage: { input: 10, output: 20 },
|
||||
stopReason: 'stop',
|
||||
} as any
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
})
|
||||
|
||||
if (mode === 'temporal') {
|
||||
console.log('Starting background workers for Temporal E2E tests...')
|
||||
agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, {
|
||||
workflowsPath: agentWorkflowsPath,
|
||||
})
|
||||
transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, {
|
||||
workflowsPath: transcodeWorkflowsPath,
|
||||
})
|
||||
|
||||
// Wait a moment for workers to establish connections
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} else {
|
||||
console.log('Starting local workflow service polling...')
|
||||
workflowService.start()
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (mode === 'temporal') {
|
||||
console.log('Shutting down Temporal workers...')
|
||||
await workflowService.shutdownWorkers()
|
||||
await Promise.all([agentWorkerPromise, transcodeWorkerPromise].filter(Boolean))
|
||||
}
|
||||
|
||||
workflowService.close()
|
||||
|
||||
// Restore spied methods
|
||||
vi.restoreAllMocks()
|
||||
|
||||
// Clean up E2E files in local filesystem storage
|
||||
try {
|
||||
console.log('Cleaning up local E2E storage files...')
|
||||
await s3Service.deletePrefix('shumai-e2e-test-bucket', '')
|
||||
} catch (err) {
|
||||
console.error('Failed to clean up E2E storage folder:', err)
|
||||
}
|
||||
})
|
||||
|
||||
it('should run agentAutofillMedia workflow and update asset metadata successfully', async () => {
|
||||
// 1. Seed Database Models
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Test Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Test Project', teamId: team.id },
|
||||
})
|
||||
|
||||
// Create Metadata Field (aiAutofill = true, text)
|
||||
await prisma.metadataField.create({
|
||||
data: {
|
||||
key: 'title',
|
||||
scope: 'PROJECT',
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
config: { name: 'Title', type: 'text' },
|
||||
aiAutofill: true,
|
||||
description: 'Auto-extracted title',
|
||||
},
|
||||
})
|
||||
|
||||
// Create Metadata Field (aiAutofill = true, number)
|
||||
await prisma.metadataField.create({
|
||||
data: {
|
||||
key: 'confidence',
|
||||
scope: 'PROJECT',
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
config: { name: 'Confidence', type: 'number' },
|
||||
aiAutofill: true,
|
||||
description: 'AI extraction confidence',
|
||||
},
|
||||
})
|
||||
|
||||
// Create Metadata Field (aiAutofill = true, boolean)
|
||||
await prisma.metadataField.create({
|
||||
data: {
|
||||
key: 'completed',
|
||||
scope: 'PROJECT',
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
config: { name: 'Completed', type: 'toggle' },
|
||||
aiAutofill: true,
|
||||
description: 'Completed flag',
|
||||
},
|
||||
})
|
||||
|
||||
// Create Metadata Field (aiAutofill = false, text)
|
||||
await prisma.metadataField.create({
|
||||
data: {
|
||||
key: 'manual_notes',
|
||||
scope: 'PROJECT',
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
config: { name: 'Manual Notes', type: 'text' },
|
||||
aiAutofill: false,
|
||||
description: 'Manual notes (should not be auto-filled)',
|
||||
},
|
||||
})
|
||||
|
||||
// Create Agent User
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E Autofill Agent User',
|
||||
email: 'e2e-autofill-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
// Link Agent User to Team
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
// Setup Agent Provider & Model configuration
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text', 'image'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'autofill',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// Create StorageKey first
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/test-image.png',
|
||||
},
|
||||
})
|
||||
|
||||
// Create Asset
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test-image.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- partial MediaInfo mock for testing
|
||||
media: { proxyType: 'image' } as any,
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage (LocalStorageService) with a valid 1x1 base64-encoded PNG image
|
||||
const base64Png =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='
|
||||
const pngBuffer = Buffer.from(base64Png, 'base64')
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket',
|
||||
'projects/e2e/test-image.png',
|
||||
pngBuffer,
|
||||
pngBuffer.length,
|
||||
'image/png',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task (ai_metadata_autofill, pending)
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_metadata_autofill',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: { sessionId: 'mock-session-123', agentId: agent.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(`Submitted E2E Workflow Task. ID: ${task.id}. Awaiting completion...`)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify placeholder comment is updated correctly
|
||||
const comments = await prisma.assetComment.findMany({
|
||||
where: { assetId: asset.id },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
})
|
||||
expect(comments.length).toBeGreaterThan(0)
|
||||
// The last updated message of placeholder comment should indicate success
|
||||
const finalComment = comments[comments.length - 1]
|
||||
expect(finalComment.message).toBe('Autofill completed successfully.')
|
||||
|
||||
// Verify AssetMetadataValue contains the AI-extracted title
|
||||
const titleVal = await prisma.assetMetadataValue.findUnique({
|
||||
where: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
assetId_fieldKey: {
|
||||
assetId: asset.id,
|
||||
fieldKey: 'title',
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(titleVal).toBeDefined()
|
||||
expect(titleVal?.stringValue).toBe('E2E Title Extracted')
|
||||
|
||||
// Verify AssetMetadataValue contains the AI-extracted confidence
|
||||
const confidenceVal = await prisma.assetMetadataValue.findUnique({
|
||||
where: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
assetId_fieldKey: {
|
||||
assetId: asset.id,
|
||||
fieldKey: 'confidence',
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(confidenceVal).toBeDefined()
|
||||
expect(confidenceVal?.numberValue).toBe(0.95)
|
||||
|
||||
// Verify AssetMetadataValue contains the AI-extracted completed flag
|
||||
const completedVal = await prisma.assetMetadataValue.findUnique({
|
||||
where: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
assetId_fieldKey: {
|
||||
assetId: asset.id,
|
||||
fieldKey: 'completed',
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(completedVal).toBeDefined()
|
||||
expect(completedVal?.booleanValue).toBe(true)
|
||||
|
||||
// Verify manual_notes is NOT auto-filled (remains null/undefined in DB)
|
||||
const manualNotesVal = await prisma.assetMetadataValue.findUnique({
|
||||
where: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
assetId_fieldKey: {
|
||||
assetId: asset.id,
|
||||
fieldKey: 'manual_notes',
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(manualNotesVal).toBeNull()
|
||||
}, 50000)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,457 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService, TaskQueueAgent } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { AgentHarness } from '@earendil-works/pi-agent-core'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts')
|
||||
|
||||
describe.each(['local', 'temporal'] as const)('Workflow E2E - agentChat (executor: %s)', (mode) => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let agentWorkerPromise: Promise<void> | null = null
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.S3_BUCKET = 'shumai-e2e-test-bucket-chat'
|
||||
process.env.GEMINI_API_KEY = 'dummy-key'
|
||||
|
||||
workflowService.setExecutorType(mode)
|
||||
initAgentWorkflows()
|
||||
|
||||
vi.spyOn(AgentHarness.prototype, 'prompt').mockResolvedValue({
|
||||
content: [{ type: 'text', text: 'E2E Agent Chat Response Success' }],
|
||||
usage: { input: 10, output: 20 },
|
||||
stopReason: 'stop',
|
||||
} as unknown as Awaited<ReturnType<typeof AgentHarness.prototype.prompt>>)
|
||||
|
||||
if (mode === 'temporal') {
|
||||
console.log('Starting background worker for chat Temporal E2E tests...')
|
||||
agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, {
|
||||
workflowsPath: agentWorkflowsPath,
|
||||
})
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} else {
|
||||
console.log('Starting local workflow service polling...')
|
||||
workflowService.start()
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (mode === 'temporal') {
|
||||
console.log('Shutting down Temporal workers...')
|
||||
await workflowService.shutdownWorkers()
|
||||
await Promise.all([agentWorkerPromise].filter(Boolean))
|
||||
}
|
||||
workflowService.close()
|
||||
vi.restoreAllMocks()
|
||||
try {
|
||||
console.log('Cleaning up local E2E storage files...')
|
||||
await s3Service.deletePrefix('shumai-e2e-test-bucket-chat', '')
|
||||
} catch (err) {
|
||||
console.error('Failed to clean up E2E storage folder:', err)
|
||||
}
|
||||
})
|
||||
|
||||
it('should run agentChat workflow with userCommentId and reply comment successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Chat Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Chat Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E Chat Agent User',
|
||||
email: 'e2e-chat-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text', 'image'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'chat-asset.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Create a regular user who authors the comment
|
||||
const regularUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Regular User',
|
||||
email: 'user@example.com',
|
||||
type: 'human',
|
||||
},
|
||||
})
|
||||
|
||||
// Create AgentSession first
|
||||
await prisma.agentSession.create({
|
||||
data: {
|
||||
id: 'chat-session-123',
|
||||
agentId: agentUser.id,
|
||||
userId: regularUser.id,
|
||||
cwd: process.cwd(),
|
||||
assetId: asset.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Create the User Comment
|
||||
const userComment = await prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
message: 'Hello E2E Agent!',
|
||||
creatorId: regularUser.id,
|
||||
sessionId: 'chat-session-123',
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: {
|
||||
agentId: agentUser.id,
|
||||
sessionId: 'chat-session-123',
|
||||
userCommentId: userComment.id,
|
||||
userId: regularUser.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 3. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Chat Workflow Task with Comment. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 4. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify reply comment in DB
|
||||
const comments = await prisma.assetComment.findMany({
|
||||
where: {
|
||||
assetId: asset.id,
|
||||
replyToId: userComment.id,
|
||||
},
|
||||
})
|
||||
expect(comments.length).toBeGreaterThan(0)
|
||||
expect(comments[0].message).toBe('E2E Agent Chat Response Success')
|
||||
expect(comments[0].creatorId).toBe(agentUser.id)
|
||||
}, 50000)
|
||||
|
||||
it('should run agentChat workflow with direct prompt successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Direct Chat Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Direct Chat Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E Direct Chat Agent User',
|
||||
email: 'e2e-dir-chat-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'direct-chat-asset.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
const regularUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Regular User 2',
|
||||
email: 'user2@example.com',
|
||||
type: 'human',
|
||||
},
|
||||
})
|
||||
|
||||
// Create AgentSession first
|
||||
await prisma.agentSession.create({
|
||||
data: {
|
||||
id: 'direct-session-456',
|
||||
agentId: agentUser.id,
|
||||
userId: regularUser.id,
|
||||
cwd: process.cwd(),
|
||||
assetId: asset.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Create Workflow Task (without userCommentId, but with prompt/sessionId)
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: {
|
||||
agentId: agentUser.id,
|
||||
sessionId: 'direct-session-456',
|
||||
prompt: 'Direct test message',
|
||||
userId: regularUser.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 3. Wait for workflow to complete
|
||||
console.log(`Submitted E2E Direct Chat Workflow Task. ID: ${task.id}. Awaiting completion...`)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 4. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
expect((completedTask.output as unknown as { sessionId: string })?.sessionId).toBe(
|
||||
'direct-session-456',
|
||||
)
|
||||
}, 50000)
|
||||
|
||||
it('should run agentChat workflow with version stack asset successfully', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Version Stack Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Version Stack Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E VS Agent User',
|
||||
email: 'e2e-vs-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text', 'image'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const versionStack = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'VersionStack Container',
|
||||
type: 'version_stack',
|
||||
status: 'uploaded',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'version-1.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
parentId: versionStack.id,
|
||||
sortIndex: 'a0',
|
||||
},
|
||||
})
|
||||
|
||||
const regularUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Regular User 3',
|
||||
email: 'user3@example.com',
|
||||
type: 'human',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agentSession.create({
|
||||
data: {
|
||||
id: 'vs-session-789',
|
||||
agentId: agentUser.id,
|
||||
userId: regularUser.id,
|
||||
cwd: process.cwd(),
|
||||
assetId: versionStack.id,
|
||||
},
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: versionStack.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: {
|
||||
agentId: agentUser.id,
|
||||
sessionId: 'vs-session-789',
|
||||
prompt: 'Explain this version stack file',
|
||||
userId: regularUser.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
expect(completedTask.status).toBe('completed')
|
||||
expect((completedTask.output as unknown as { sessionId: string })?.sessionId).toBe(
|
||||
'vs-session-789',
|
||||
)
|
||||
}, 50000)
|
||||
})
|
||||
@@ -0,0 +1,481 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { prisma, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService, TaskQueueAgent, TaskQueueTranscode } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
|
||||
const mockEmbedContent = vi.fn().mockResolvedValue({
|
||||
embeddings: [
|
||||
{
|
||||
values: Array(1536).fill(0.2),
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
vi.mock('@google/genai', () => {
|
||||
return {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
GoogleGenAI: class {
|
||||
models = {
|
||||
embedContent: mockEmbedContent,
|
||||
}
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts')
|
||||
const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts')
|
||||
const fixturesDir = path.resolve(currentDir, '../fixtures')
|
||||
|
||||
describe.each(['local', 'temporal'] as const)(
|
||||
'Workflow E2E - agentEmbeddingMedia (executor: %s)',
|
||||
(mode) => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let agentWorkerPromise: Promise<void> | null = null
|
||||
let transcodeWorkerPromise: Promise<void> | null = null
|
||||
|
||||
beforeAll(async () => {
|
||||
// Set bucket environment
|
||||
process.env.S3_BUCKET = 'shumai-e2e-test-bucket-embedding'
|
||||
process.env.GEMINI_API_KEY = 'dummy-key'
|
||||
|
||||
// Configure the workflow service executor dynamically
|
||||
workflowService.setExecutorType(mode)
|
||||
|
||||
// Initialize registries
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
|
||||
if (mode === 'temporal') {
|
||||
console.log('Starting background workers for Temporal E2E tests...')
|
||||
agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, {
|
||||
workflowsPath: agentWorkflowsPath,
|
||||
})
|
||||
transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, {
|
||||
workflowsPath: transcodeWorkflowsPath,
|
||||
})
|
||||
|
||||
// Wait a moment for workers to establish connections
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} else {
|
||||
console.log('Starting local workflow service polling...')
|
||||
workflowService.start()
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (mode === 'temporal') {
|
||||
console.log('Shutting down Temporal workers...')
|
||||
await workflowService.shutdownWorkers()
|
||||
await Promise.all([agentWorkerPromise, transcodeWorkerPromise].filter(Boolean))
|
||||
}
|
||||
|
||||
workflowService.close()
|
||||
|
||||
// Restore spied methods
|
||||
vi.restoreAllMocks()
|
||||
|
||||
// Clean up E2E files in local filesystem storage
|
||||
try {
|
||||
console.log('Cleaning up local E2E storage files...')
|
||||
await s3Service.deletePrefix('shumai-e2e-test-bucket-embedding', '')
|
||||
} catch (err) {
|
||||
console.error('Failed to clean up E2E storage folder:', err)
|
||||
}
|
||||
})
|
||||
|
||||
it('should run agentEmbeddingMedia workflow for an image asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Image Embedding Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Image Embedding Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E Image Embedding Agent User',
|
||||
email: 'e2e-img-emb-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text', 'image'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/test-image.png',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test-image.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const pngPath = path.join(fixturesDir, 'small.png')
|
||||
const pngBuffer = fs.readFileSync(pngPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-embedding',
|
||||
'projects/e2e/test-image.png',
|
||||
pngBuffer,
|
||||
pngBuffer.length,
|
||||
'image/png',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task (ai_embedding, pending)
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: { agentId: agentUser.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Image Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify AssetComment indicates success
|
||||
const comments = await prisma.assetComment.findMany({
|
||||
where: { assetId: asset.id },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
})
|
||||
expect(comments.length).toBeGreaterThan(0)
|
||||
const finalComment = comments[comments.length - 1]
|
||||
expect(finalComment.message).toBe('Embedding completed successfully.')
|
||||
|
||||
// Verify raw embedding is saved in DB
|
||||
const dbEmbeddings = (await prisma.$queryRaw`
|
||||
SELECT id, asset_id as "assetId", start_time as "startTime", end_time as "endTime"
|
||||
FROM asset_embeddings
|
||||
WHERE asset_id = ${asset.id}
|
||||
`) as { id: string; startTime: number | null; endTime: number | null }[]
|
||||
expect(dbEmbeddings.length).toBe(1)
|
||||
expect(dbEmbeddings[0].startTime).toBeNull()
|
||||
expect(dbEmbeddings[0].endTime).toBeNull()
|
||||
}, 50000)
|
||||
|
||||
it('should run agentEmbeddingMedia workflow for a video asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Video Embedding Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Video Embedding Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E Video Embedding Agent User',
|
||||
email: 'e2e-vid-emb-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text', 'image'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/test-video.mp4',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test-video.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
media: { duration: 1.0 } as unknown as PrismaJson.MediaInfo, // 1-second video
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const mp4Path = path.join(fixturesDir, 'small.mp4')
|
||||
const mp4Buffer = fs.readFileSync(mp4Path)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-embedding',
|
||||
'projects/e2e/test-video.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task (ai_embedding, pending)
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: { agentId: agentUser.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Video Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify raw embedding is saved in DB with start and end times
|
||||
const dbEmbeddings = (await prisma.$queryRaw`
|
||||
SELECT id, asset_id as "assetId", start_time as "startTime", end_time as "endTime"
|
||||
FROM asset_embeddings
|
||||
WHERE asset_id = ${asset.id}
|
||||
ORDER BY start_time ASC
|
||||
`) as { id: string; startTime: number | null; endTime: number | null }[]
|
||||
expect(dbEmbeddings.length).toBeGreaterThan(0)
|
||||
expect(dbEmbeddings[0].startTime).toBe(0)
|
||||
expect(dbEmbeddings[0].endTime).toBe(1)
|
||||
}, 50000)
|
||||
|
||||
it('should fail agentEmbeddingMedia workflow for an unsupported audio asset', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Audio Embedding Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Audio Embedding Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'E2E Audio Embedding Agent User',
|
||||
email: 'e2e-aud-emb-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: agentUser.id,
|
||||
role: 'editor',
|
||||
},
|
||||
})
|
||||
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'google',
|
||||
teamId: team.id,
|
||||
config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' },
|
||||
},
|
||||
})
|
||||
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'gemini',
|
||||
name: 'Gemini',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
input: ['text'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 2048,
|
||||
reasoning: false,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/test-audio.wav',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test-audio.wav',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'audio/wav',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const wavPath = path.join(fixturesDir, 'small.wav')
|
||||
const wavBuffer = fs.readFileSync(wavPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-embedding',
|
||||
'projects/e2e/test-audio.wav',
|
||||
wavBuffer,
|
||||
wavBuffer.length,
|
||||
'audio/wav',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task (ai_embedding, pending)
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agent: { agentId: agentUser.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to fail and verify error message
|
||||
console.log(
|
||||
`Submitted E2E Audio Embedding Workflow Task. ID: ${task.id}. Awaiting failure...`,
|
||||
)
|
||||
await expect(workflowService.executeWait(task, 45000)).rejects.toThrow(
|
||||
'unsupported media type for embeddings: audio/wav',
|
||||
)
|
||||
|
||||
// Verify task status in DB is failed
|
||||
const dbTask = await prisma.workflowTask.findUnique({
|
||||
where: { id: task.id },
|
||||
})
|
||||
expect(dbTask?.status).toBe(WorkflowTaskStatus.failed)
|
||||
}, 50000)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,146 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService, TaskQueueAgent } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts')
|
||||
|
||||
describe.each(['local', 'temporal'] as const)(
|
||||
'Workflow E2E - agentToolCall (executor: %s)',
|
||||
(mode) => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let agentWorkerPromise: Promise<void> | null = null
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.S3_BUCKET = 'shumai-e2e-test-bucket-toolcall'
|
||||
process.env.GEMINI_API_KEY = 'dummy-key'
|
||||
|
||||
workflowService.setExecutorType(mode)
|
||||
initAgentWorkflows()
|
||||
|
||||
if (mode === 'temporal') {
|
||||
console.log('Starting background worker for tool call Temporal E2E tests...')
|
||||
agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, {
|
||||
workflowsPath: agentWorkflowsPath,
|
||||
})
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} else {
|
||||
console.log('Starting local workflow service polling...')
|
||||
workflowService.start()
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (mode === 'temporal') {
|
||||
console.log('Shutting down Temporal workers...')
|
||||
await workflowService.shutdownWorkers()
|
||||
await Promise.all([agentWorkerPromise].filter(Boolean))
|
||||
}
|
||||
workflowService.close()
|
||||
vi.restoreAllMocks()
|
||||
try {
|
||||
console.log('Cleaning up local E2E storage files...')
|
||||
await s3Service.deletePrefix('shumai-e2e-test-bucket-toolcall', '')
|
||||
} catch (err) {
|
||||
console.error('Failed to clean up E2E storage folder:', err)
|
||||
}
|
||||
})
|
||||
|
||||
it('should run agentToolCall workflow and execute list_assets successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Tool Call Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Tool Call Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const regularUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Regular Editor User',
|
||||
email: 'editor@example.com',
|
||||
type: 'human',
|
||||
},
|
||||
})
|
||||
|
||||
// Setup team member with owner/editor role and team scope
|
||||
await prisma.teamMember.create({
|
||||
data: {
|
||||
teamId: team.id,
|
||||
userId: regularUser.id,
|
||||
role: 'editor',
|
||||
scope: 'team',
|
||||
},
|
||||
})
|
||||
|
||||
// Create a folder asset (parent)
|
||||
const parentFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'TestParentFolder',
|
||||
type: 'folder',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Create a child file asset inside the parent folder
|
||||
const childFile = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'TestChildFile.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
parentId: parentFolder.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'agent_tool_call',
|
||||
status: 'pending',
|
||||
assetId: parentFolder.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
agentToolCall: {
|
||||
toolName: 'list_assets',
|
||||
args: {
|
||||
parent: parentFolder.id,
|
||||
page: 1,
|
||||
pageSize: 20,
|
||||
type: 'all',
|
||||
},
|
||||
userId: regularUser.id,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 3. Wait for workflow to complete
|
||||
console.log(`Submitted E2E Tool Call Workflow Task. ID: ${task.id}. Awaiting completion...`)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 4. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify task output contains the list of files, and childFile.id is present in assets
|
||||
const output = completedTask.output as unknown as { assets: { id: string }[] }
|
||||
expect(output).toBeDefined()
|
||||
expect(output.assets).toBeDefined()
|
||||
expect(output.assets.length).toBeGreaterThan(0)
|
||||
|
||||
const assetIds = output.assets.map((a) => a.id)
|
||||
expect(assetIds).toContain(childFile.id)
|
||||
}, 50000)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService, TaskQueueAgent } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
const mockEmbedContent = vi.fn().mockResolvedValue({
|
||||
embeddings: [
|
||||
{
|
||||
values: Array(1536).fill(0.35),
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
vi.mock('@google/genai', () => {
|
||||
return {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
GoogleGenAI: class {
|
||||
models = {
|
||||
embedContent: mockEmbedContent,
|
||||
}
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts')
|
||||
|
||||
describe.each(['local', 'temporal'] as const)(
|
||||
'Workflow E2E - queryEmbeddingForSearch (executor: %s)',
|
||||
(mode) => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let agentWorkerPromise: Promise<void> | null = null
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.S3_BUCKET = 'shumai-e2e-test-bucket-query'
|
||||
process.env.GEMINI_API_KEY = 'dummy-key'
|
||||
|
||||
workflowService.setExecutorType(mode)
|
||||
initAgentWorkflows()
|
||||
|
||||
if (mode === 'temporal') {
|
||||
console.log('Starting background worker for query embedding Temporal E2E tests...')
|
||||
agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, {
|
||||
workflowsPath: agentWorkflowsPath,
|
||||
})
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} else {
|
||||
console.log('Starting local workflow service polling...')
|
||||
workflowService.start()
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (mode === 'temporal') {
|
||||
console.log('Shutting down Temporal workers...')
|
||||
await workflowService.shutdownWorkers()
|
||||
await Promise.all([agentWorkerPromise].filter(Boolean))
|
||||
}
|
||||
workflowService.close()
|
||||
vi.restoreAllMocks()
|
||||
try {
|
||||
console.log('Cleaning up local E2E storage files...')
|
||||
await s3Service.deletePrefix('shumai-e2e-test-bucket-query', '')
|
||||
} catch (err) {
|
||||
console.error('Failed to clean up E2E storage folder:', err)
|
||||
}
|
||||
})
|
||||
|
||||
it('should run queryEmbeddingForSearch workflow and return embedding successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Query Embedding Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Query Embedding Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'dummy-search-asset',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'text/plain',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'query_embedding_for_search',
|
||||
status: 'pending',
|
||||
teamId: team.id,
|
||||
projectId: project.id,
|
||||
assetId: asset.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
queryEmbeddingForSearch: {
|
||||
text: 'test search query',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 3. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Query Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 4. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify output payload contains correct embedding values and usage metadata
|
||||
const output = completedTask.output as unknown as { embedding: number[] }
|
||||
expect(output).toBeDefined()
|
||||
expect(output.embedding).toEqual(Array(1536).fill(0.35))
|
||||
expect(completedTask.model).toBe('gemini-embedding-2')
|
||||
}, 50000)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
import { setup as dbSetup, teardown as dbTeardown } from '../../db/src/test-global-setup'
|
||||
import { GenericContainer, StartedTestContainer } from 'testcontainers'
|
||||
|
||||
let temporalContainer: StartedTestContainer | null = null
|
||||
|
||||
export async function setup() {
|
||||
// 1. Start Postgres DB container & run migrations
|
||||
await dbSetup()
|
||||
|
||||
// 2. Start Temporal Dev Server container for the test suite
|
||||
console.log('Starting Temporal Dev Server container using testcontainers...')
|
||||
try {
|
||||
temporalContainer = await new GenericContainer('temporalio/temporal:latest')
|
||||
.withExposedPorts(7233, 8233)
|
||||
.withCommand(['server', 'start-dev', '--ip', '0.0.0.0'])
|
||||
.start()
|
||||
|
||||
const host = temporalContainer.getHost()
|
||||
const mappedPort = temporalContainer.getMappedPort(7233)
|
||||
const address = `${host}:${mappedPort}`
|
||||
|
||||
process.env.TEMPORAL_ADDRESS = address
|
||||
console.log(`Temporal Dev Server container started. Address: ${address}`)
|
||||
|
||||
// Brief sleep to ensure temporal dev server is fully ready to accept gRPC clients
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} catch (err) {
|
||||
console.error('Failed to start Temporal Dev Server container:', err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export async function teardown() {
|
||||
// 1. Stop Temporal container if running
|
||||
if (temporalContainer) {
|
||||
console.log('Stopping Temporal Dev Server container...')
|
||||
try {
|
||||
await temporalContainer.stop()
|
||||
} catch (err) {
|
||||
console.error('Failed to stop Temporal Dev Server container:', err)
|
||||
}
|
||||
temporalContainer = null
|
||||
}
|
||||
|
||||
// 2. Stop Postgres DB container
|
||||
await dbTeardown()
|
||||
}
|
||||
@@ -0,0 +1,790 @@
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
|
||||
import { prisma, AssetStatus } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService, TaskQueueTranscode } from '@shumai/workflow-core'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts')
|
||||
const fixturesDir = path.resolve(currentDir, '../fixtures')
|
||||
|
||||
describe.each(['local', 'temporal'] as const)(
|
||||
'Workflow E2E - transcodeMedia (executor: %s)',
|
||||
(mode) => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let transcodeWorkerPromise: Promise<void> | null = null
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.S3_BUCKET = 'shumai-e2e-test-bucket-transcode'
|
||||
|
||||
workflowService.setExecutorType(mode)
|
||||
initTranscodeWorkflows()
|
||||
|
||||
if (mode === 'temporal') {
|
||||
console.log('Starting background worker for transcode Temporal E2E tests...')
|
||||
transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, {
|
||||
workflowsPath: transcodeWorkflowsPath,
|
||||
})
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
} else {
|
||||
console.log('Starting local workflow service polling...')
|
||||
workflowService.start()
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (mode === 'temporal') {
|
||||
console.log('Shutting down Temporal workers...')
|
||||
await workflowService.shutdownWorkers()
|
||||
await Promise.all([transcodeWorkerPromise].filter(Boolean))
|
||||
}
|
||||
workflowService.close()
|
||||
vi.restoreAllMocks()
|
||||
try {
|
||||
console.log('Cleaning up local E2E storage files...')
|
||||
await s3Service.deletePrefix('shumai-e2e-test-bucket-transcode', '')
|
||||
} catch (err) {
|
||||
console.error('Failed to clean up E2E storage folder:', err)
|
||||
}
|
||||
})
|
||||
|
||||
it('should run transcodeMedia workflow for a video asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Video Transcode Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Video Transcode Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/video-trans.mp4',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video-trans.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const mp4Path = path.join(fixturesDir, 'small.mp4')
|
||||
const mp4Buffer = fs.readFileSync(mp4Path)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/video-trans.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
thumbnail: false,
|
||||
poster: true,
|
||||
sprite: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Video Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify Asset status is updated to processed
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
// Verify media info contains videoTranscodes, poster, sprite, and duration details
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
duration: number
|
||||
videoTranscodes: { key: string }[]
|
||||
poster: unknown
|
||||
sprite: unknown
|
||||
}
|
||||
expect(mediaInfo).toBeDefined()
|
||||
expect(mediaInfo.proxyType).toBe('video')
|
||||
expect(mediaInfo.duration).toBeCloseTo(1.0, 1)
|
||||
expect(mediaInfo.videoTranscodes).toBeDefined()
|
||||
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
|
||||
expect(mediaInfo.poster).toBeDefined()
|
||||
expect(mediaInfo.sprite).toBeDefined()
|
||||
}, 50000)
|
||||
|
||||
it('should run transcodeMedia workflow for an image asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Image Transcode Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Image Transcode Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/image-trans.png',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'image-trans.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const pngPath = path.join(fixturesDir, 'small.png')
|
||||
const pngBuffer = fs.readFileSync(pngPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/image-trans.png',
|
||||
pngBuffer,
|
||||
pngBuffer.length,
|
||||
'image/png',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_image',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
thumbnail: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Image Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
imageTranscodes: unknown[]
|
||||
thumbnail: unknown
|
||||
}
|
||||
expect(mediaInfo).toBeDefined()
|
||||
expect(mediaInfo.proxyType).toBe('image')
|
||||
expect(mediaInfo.imageTranscodes).toBeDefined()
|
||||
expect(mediaInfo.imageTranscodes.length).toBeGreaterThan(0)
|
||||
expect(mediaInfo.thumbnail).toBeDefined()
|
||||
}, 50000)
|
||||
|
||||
it('should run transcodeMedia workflow for an audio asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Audio Transcode Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Audio Transcode Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/audio-trans.wav',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'audio-trans.wav',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'audio/wav',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const wavPath = path.join(fixturesDir, 'small.wav')
|
||||
const wavBuffer = fs.readFileSync(wavPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/audio-trans.wav',
|
||||
wavBuffer,
|
||||
wavBuffer.length,
|
||||
'audio/wav',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Audio Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
videoTranscodes: { key: string }[]
|
||||
}
|
||||
expect(mediaInfo).toBeDefined()
|
||||
expect(mediaInfo.proxyType).toBe('audio')
|
||||
expect(mediaInfo.videoTranscodes).toBeDefined()
|
||||
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
|
||||
// Audio proxy key should end with -audio-proxy.mp4
|
||||
expect(mediaInfo.videoTranscodes[0].key).toContain('-audio-proxy.mp4')
|
||||
}, 50000)
|
||||
|
||||
it('should run transcodeMedia screenshot extraction successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Video Screenshot Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Video Screenshot Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/video-shot.mp4',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video-shot.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const mp4Path = path.join(fixturesDir, 'small.mp4')
|
||||
const mp4Buffer = fs.readFileSync(mp4Path)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/video-shot.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_screenshot',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
screenshot: {
|
||||
start: 0,
|
||||
end: 1,
|
||||
count: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Video Screenshot Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify task output contains list of screenshots
|
||||
const output = completedTask.output as unknown as { screenshots: { key: string }[] }
|
||||
expect(output).toBeDefined()
|
||||
expect(output.screenshots).toBeDefined()
|
||||
expect(output.screenshots.length).toBe(1)
|
||||
expect(output.screenshots[0].key).toContain('screenshots/')
|
||||
}, 50000)
|
||||
|
||||
it('should run transcodeMedia image annotation overlay successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Image Annotation Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Image Annotation Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/image-ann.png',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'image-ann.png',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from Fixture
|
||||
const pngPath = path.join(fixturesDir, 'small.png')
|
||||
const pngBuffer = fs.readFileSync(pngPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/image-ann.png',
|
||||
pngBuffer,
|
||||
pngBuffer.length,
|
||||
'image/png',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_image_annotation',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
imageAnnotation: {
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0.1, 0.1],
|
||||
[0.9, 0.9],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E Image Annotation Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify task output contains overlay image key
|
||||
const output = completedTask.output as unknown as { key: string }
|
||||
expect(output).toBeDefined()
|
||||
expect(output.key).toContain('annotations/')
|
||||
}, 50000)
|
||||
|
||||
it('should run transcodeMedia workflow for a PDF asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E PDF Transcode Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E PDF Transcode Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/test.pdf',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test.pdf',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'application/pdf',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from fixture test.pdf
|
||||
const pdfPath = path.join(fixturesDir, 'test.pdf')
|
||||
const pdfBuffer = fs.readFileSync(pdfPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/test.pdf',
|
||||
pdfBuffer,
|
||||
pdfBuffer.length,
|
||||
'application/pdf',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_pdf',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
poster: true,
|
||||
sprite: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E PDF Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify Asset status is updated to processed
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
// Verify media info contains pdf details, poster, sprite, and pageCount
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
poster: { key: string }
|
||||
sprite: { key: string }
|
||||
frames?: number
|
||||
pageCount?: number
|
||||
metadata?: { totalFrames?: number; pageCount?: number }
|
||||
}
|
||||
expect(mediaInfo).toBeDefined()
|
||||
expect(mediaInfo.proxyType).toBe('pdf')
|
||||
expect(mediaInfo.poster).toBeDefined()
|
||||
expect(mediaInfo.poster.key).toContain('poster.webp')
|
||||
expect(mediaInfo.sprite).toBeDefined()
|
||||
expect(mediaInfo.sprite.key).toContain('sprite.webp')
|
||||
expect(mediaInfo.frames ?? mediaInfo.metadata?.totalFrames).toBeGreaterThan(0)
|
||||
}, 50000)
|
||||
|
||||
it('should run renderPdfPages workflow for a PDF asset successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E PDF Pages Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E PDF Pages Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/test-pages.pdf',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'test-pages.pdf',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'application/pdf',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage from fixture test.pdf
|
||||
const pdfPath = path.join(fixturesDir, 'test.pdf')
|
||||
const pdfBuffer = fs.readFileSync(pdfPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/test-pages.pdf',
|
||||
pdfBuffer,
|
||||
pdfBuffer.length,
|
||||
'application/pdf',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_pdf_pages',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
pdfPages: {
|
||||
start: 1,
|
||||
end: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(`Submitted E2E PDF Pages Workflow Task. ID: ${task.id}. Awaiting completion...`)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
// Verify output contains rendered pages
|
||||
const output = completedTask.output as unknown as { pages: { key: string; page: number }[] }
|
||||
expect(output).toBeDefined()
|
||||
expect(output.pages).toBeDefined()
|
||||
expect(output.pages.length).toBeGreaterThan(0)
|
||||
}, 50000)
|
||||
|
||||
it('should run transcodeMedia workflow for a TXT asset with CJK text to generate PDF proxy successfully', async () => {
|
||||
// 1. Seed Database
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E CJK TXT Transcode Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E CJK TXT Transcode Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/cjk-test.txt',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'cjk-test.txt',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'text/plain',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage with CJK text
|
||||
const cjkText =
|
||||
'Hello World\n你好世界 (Chinese)\nこんにちは世界 (Japanese)\n안녕하세요世界 (Korean)\nLine 5 of random text.'
|
||||
const txtBuffer = Buffer.from(cjkText, 'utf-8')
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/cjk-test.txt',
|
||||
txtBuffer,
|
||||
txtBuffer.length,
|
||||
'text/plain',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_pdf',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
poster: true,
|
||||
sprite: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E CJK TXT Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType?: string
|
||||
pdfTranscode?: { key: string }
|
||||
poster?: { key: string }
|
||||
sprite?: { key: string }
|
||||
frames?: number
|
||||
}
|
||||
expect(mediaInfo).toBeDefined()
|
||||
expect(mediaInfo.proxyType).toBe('pdf')
|
||||
expect(mediaInfo.pdfTranscode?.key).toBe(`files/${asset.id}/proxy.pdf`)
|
||||
expect(mediaInfo.poster?.key).toContain('poster.webp')
|
||||
expect(mediaInfo.sprite?.key).toContain('sprite.webp')
|
||||
expect(mediaInfo.frames).toBeGreaterThan(0)
|
||||
}, 50000)
|
||||
|
||||
it('should process markdown (.md) transcode PDF task correctly', async () => {
|
||||
// 1. Create Team, Project, Asset & StorageKey
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E MD Transcode Team' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E MD Transcode Project', teamId: team.id },
|
||||
})
|
||||
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: {
|
||||
key: 'projects/e2e/doc.md',
|
||||
status: 'active',
|
||||
},
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'doc.md',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'text/markdown',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Seed S3 Storage with Markdown text
|
||||
const mdText = '# Heading\n\nThis is a **markdown** document to be rendered to PDF proxy.'
|
||||
const mdBuffer = Buffer.from(mdText, 'utf-8')
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/doc.md',
|
||||
mdBuffer,
|
||||
mdBuffer.length,
|
||||
'text/markdown',
|
||||
)
|
||||
|
||||
// 3. Create Workflow Task
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_pdf',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
poster: true,
|
||||
sprite: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Wait for workflow to complete
|
||||
console.log(
|
||||
`Submitted E2E MD Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`,
|
||||
)
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
|
||||
// 5. Verification
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType?: string
|
||||
pdfTranscode?: { key: string }
|
||||
poster?: { key: string }
|
||||
sprite?: { key: string }
|
||||
frames?: number
|
||||
}
|
||||
expect(mediaInfo).toBeDefined()
|
||||
expect(mediaInfo.proxyType).toBe('pdf')
|
||||
expect(mediaInfo.pdfTranscode?.key).toBe(`files/${asset.id}/proxy.pdf`)
|
||||
expect(mediaInfo.poster?.key).toContain('poster.webp')
|
||||
expect(mediaInfo.sprite?.key).toContain('sprite.webp')
|
||||
expect(mediaInfo.frames).toBeGreaterThan(0)
|
||||
}, 50000)
|
||||
},
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/tableprinter",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.1.9",
|
||||
"version": "0.2.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -3,7 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import * as child_process from 'child_process'
|
||||
|
||||
vi.mock('child_process', () => ({
|
||||
@@ -12,6 +12,7 @@ vi.mock('child_process', () => ({
|
||||
import {
|
||||
getMediaInfoActivity,
|
||||
overlayAnnotationsActivity,
|
||||
renderPdfPagesActivity,
|
||||
takeScreenshotsActivity,
|
||||
transcodeVideoActivity,
|
||||
transcodeAudioActivity,
|
||||
@@ -19,6 +20,7 @@ import {
|
||||
downloadMediaToTmpActivity,
|
||||
transcodeImageActivity,
|
||||
generateSpriteActivity,
|
||||
generatePdfProxyActivity,
|
||||
transcodeVideoChunkActivity,
|
||||
deleteS3ObjectActivity,
|
||||
} from './transcode'
|
||||
@@ -39,14 +41,32 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({
|
||||
getVideoInfo: vi.fn(),
|
||||
getAudioInfo: vi.fn(),
|
||||
getImageInfo: vi.fn(),
|
||||
getPdfInfo: vi.fn().mockResolvedValue({
|
||||
originalWidth: 800,
|
||||
originalHeight: 1000,
|
||||
duration: 0,
|
||||
bitRate: 0,
|
||||
frameRate: 0,
|
||||
totalFrames: 10,
|
||||
hasAudio: false,
|
||||
mimeType: 'application/pdf',
|
||||
}),
|
||||
transcodeVideo: vi.fn(),
|
||||
transcodeAudio: vi.fn(),
|
||||
transcodeImage: vi.fn(),
|
||||
generateSprite: vi.fn(),
|
||||
generatePdfSprite: vi.fn().mockResolvedValue({
|
||||
pageCount: 10,
|
||||
originalWidth: 800,
|
||||
originalHeight: 1000,
|
||||
}),
|
||||
generatePdfFromText: vi.fn(),
|
||||
generatePdfFromCsv: vi.fn(),
|
||||
createTempDir: vi.fn().mockReturnValue('/tmp'),
|
||||
removeDir: vi.fn(),
|
||||
takeScreenshots: vi.fn(),
|
||||
overlayAnnotations: vi.fn(),
|
||||
renderPdfPages: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
@@ -97,6 +117,7 @@ describe('Transcode Activities', () => {
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/v.mp4',
|
||||
proxyType: 'video',
|
||||
mediaType: 'video/mp4',
|
||||
})
|
||||
|
||||
@@ -133,6 +154,7 @@ describe('Transcode Activities', () => {
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/i.png',
|
||||
proxyType: 'image',
|
||||
mediaType: 'image/png',
|
||||
})
|
||||
|
||||
@@ -168,6 +190,7 @@ describe('Transcode Activities', () => {
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/a.mp3',
|
||||
proxyType: 'audio',
|
||||
mediaType: 'audio/mpeg',
|
||||
})
|
||||
|
||||
@@ -201,16 +224,73 @@ describe('Transcode Activities', () => {
|
||||
await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/d.pdf',
|
||||
proxyType: 'pdf',
|
||||
mediaType: 'application/pdf',
|
||||
})
|
||||
|
||||
expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith(
|
||||
asset.id,
|
||||
expect.arrayContaining([{ key: 'file_type', value: 'document' }]),
|
||||
expect.arrayContaining([
|
||||
{ key: 'file_type', value: 'document' },
|
||||
{ key: 'proxy_type', value: 'pdf' },
|
||||
]),
|
||||
true,
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate PDF proxy for text/csv files in generatePdfProxyActivity', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'test.txt', type: 'file', status: 'uploaded' },
|
||||
})
|
||||
|
||||
const generatePdfFromTextSpy = vi
|
||||
.spyOn(transcodeService, 'generatePdfFromText')
|
||||
.mockImplementation(async (_inPath, outPath) => {
|
||||
const fs = await import('fs')
|
||||
fs.writeFileSync(outPath, 'fake pdf content')
|
||||
})
|
||||
|
||||
const res = await generatePdfProxyActivity({
|
||||
assetId: asset.id,
|
||||
assetKey: 'files/asset1/test.txt',
|
||||
filePath: '/tmp/test.txt',
|
||||
mediaType: 'text/plain',
|
||||
filename: 'test.txt',
|
||||
})
|
||||
|
||||
expect(res.pdfProxyKey).toBe(`files/${asset.id}/proxy.pdf`)
|
||||
expect(generatePdfFromTextSpy).toHaveBeenCalled()
|
||||
generatePdfFromTextSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('should generate PDF proxy for markdown files in generatePdfProxyActivity', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'README.md', type: 'file', status: 'uploaded' },
|
||||
})
|
||||
|
||||
const generatePdfFromTextSpy = vi
|
||||
.spyOn(transcodeService, 'generatePdfFromText')
|
||||
.mockImplementation(async (_inPath, outPath) => {
|
||||
const fs = await import('fs')
|
||||
fs.writeFileSync(outPath, 'fake pdf content from md')
|
||||
})
|
||||
|
||||
const res = await generatePdfProxyActivity({
|
||||
assetId: asset.id,
|
||||
assetKey: 'files/asset1/README.md',
|
||||
filePath: '/tmp/README.md',
|
||||
mediaType: 'text/markdown',
|
||||
filename: 'README.md',
|
||||
})
|
||||
|
||||
expect(res.pdfProxyKey).toBe(`files/${asset.id}/proxy.pdf`)
|
||||
expect(generatePdfFromTextSpy).toHaveBeenCalledWith(
|
||||
'/tmp/README.md',
|
||||
expect.stringContaining('proxy.pdf'),
|
||||
)
|
||||
generatePdfFromTextSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('should set file_type to file for unknown types', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'u.xyz', type: 'file', status: 'uploaded' },
|
||||
@@ -306,6 +386,31 @@ describe('Transcode Activities', () => {
|
||||
expect(result).toBe('annotations/ann.webp')
|
||||
})
|
||||
|
||||
it('should call transcodeService.renderPdfPages', async () => {
|
||||
vi.mocked(transcodeService.renderPdfPages).mockResolvedValue([
|
||||
{ key: 'pdf_pages/doc-page-1.webp', page: 1 },
|
||||
])
|
||||
|
||||
const result = await renderPdfPagesActivity({
|
||||
assetKey: 'doc.pdf',
|
||||
assetId: 'asset-pdf',
|
||||
start: 1,
|
||||
end: 1,
|
||||
commentTimestamp: 1,
|
||||
annotations: [],
|
||||
})
|
||||
|
||||
expect(transcodeService.renderPdfPages).toHaveBeenCalledWith({
|
||||
assetKey: 'doc.pdf',
|
||||
assetId: 'asset-pdf',
|
||||
start: 1,
|
||||
end: 1,
|
||||
commentTimestamp: 1,
|
||||
annotations: [],
|
||||
})
|
||||
expect(result).toEqual([{ key: 'pdf_pages/doc-page-1.webp', page: 1 }])
|
||||
})
|
||||
|
||||
describe('Non-retryable Error Handling', () => {
|
||||
it('should throw non-retryable ApplicationFailure when downloadMediaToTmpActivity fails with ENOENT', async () => {
|
||||
vi.mocked(s3Service.downloadToFile).mockRejectedValue({
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { stemFromKey } from '@shumai/core/src/utils/filename'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
@@ -24,7 +25,12 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
|
||||
const record = err as Record<string, unknown>
|
||||
return {
|
||||
code: typeof record.code === 'string' ? record.code : undefined,
|
||||
message: typeof record.message === 'string' ? record.message : String(err),
|
||||
message:
|
||||
typeof record.message === 'string'
|
||||
? record.message
|
||||
: err instanceof Error
|
||||
? err.message
|
||||
: String(err),
|
||||
name: typeof record.name === 'string' ? record.name : undefined,
|
||||
}
|
||||
}
|
||||
@@ -36,18 +42,16 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
|
||||
export async function getMediaInfoActivity(params: {
|
||||
filePath: string
|
||||
assetId: string
|
||||
mediaType: string
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
mediaType?: string
|
||||
}): Promise<PrismaJson.MediaInfo> {
|
||||
try {
|
||||
const isVideo = params.mediaType.startsWith('video/')
|
||||
const isImage = params.mediaType.startsWith('image/')
|
||||
const isAudio = params.mediaType.startsWith('audio/')
|
||||
const isDocument =
|
||||
params.mediaType.startsWith('text/') ||
|
||||
params.mediaType === 'application/pdf' ||
|
||||
params.mediaType.includes('msword') ||
|
||||
params.mediaType.includes('officedocument') ||
|
||||
params.mediaType.includes('vnd.ms-')
|
||||
const proxyType =
|
||||
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
|
||||
const isVideo = proxyType === 'video'
|
||||
const isImage = proxyType === 'image'
|
||||
const isAudio = proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
|
||||
const fileType = isVideo
|
||||
? 'video'
|
||||
@@ -55,7 +59,7 @@ export async function getMediaInfoActivity(params: {
|
||||
? 'audio'
|
||||
: isImage
|
||||
? 'image'
|
||||
: isDocument
|
||||
: isPdf
|
||||
? 'document'
|
||||
: 'file'
|
||||
|
||||
@@ -63,12 +67,12 @@ export async function getMediaInfoActivity(params: {
|
||||
duration: 0,
|
||||
filesize: 0,
|
||||
frames: 0,
|
||||
proxyType,
|
||||
imageTranscodes: [],
|
||||
videoTranscodes: [],
|
||||
videoPreview: { width: 0, height: 0 },
|
||||
finishedAt: new Date().toISOString(),
|
||||
metadata: null,
|
||||
mimeType: params.mediaType,
|
||||
original: {
|
||||
key: '', // Will be filled by caller or updated later
|
||||
downloadUrl: '',
|
||||
@@ -80,6 +84,9 @@ export async function getMediaInfoActivity(params: {
|
||||
const metadataUpdates: { key: string; value: string | number }[] = [
|
||||
{ key: 'file_type', value: fileType },
|
||||
]
|
||||
if (proxyType) {
|
||||
metadataUpdates.push({ key: 'proxy_type', value: proxyType })
|
||||
}
|
||||
|
||||
if (isVideo) {
|
||||
const info = await transcodeService.getVideoInfo(params.filePath)
|
||||
@@ -163,6 +170,25 @@ export async function getMediaInfoActivity(params: {
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
)
|
||||
} else if (isPdf) {
|
||||
const info = await transcodeService.getPdfInfo(params.filePath)
|
||||
mediaInfo.frames = info.totalFrames
|
||||
mediaInfo.metadata = {
|
||||
originalWidth: info.originalWidth,
|
||||
originalHeight: info.originalHeight,
|
||||
duration: 0,
|
||||
bitRate: 0,
|
||||
frameRate: 0,
|
||||
totalFrames: info.totalFrames,
|
||||
startTimecode: '00:00:00:00',
|
||||
hasAudio: false,
|
||||
format: {},
|
||||
}
|
||||
metadataUpdates.push(
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
{ key: 'total_pages', value: info.totalFrames },
|
||||
)
|
||||
}
|
||||
|
||||
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
|
||||
@@ -401,12 +427,26 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
const posterFile = path.join(tmpDir, 'poster.webp')
|
||||
|
||||
try {
|
||||
await transcodeService.generateSprite(
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
params.mediaInfo.duration,
|
||||
)
|
||||
if (params.mediaInfo.proxyType === 'pdf') {
|
||||
const pdfRes = await transcodeService.generatePdfSprite(
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
)
|
||||
if (params.mediaInfo && params.mediaInfo.metadata) {
|
||||
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
|
||||
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
|
||||
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
|
||||
params.mediaInfo.frames = pdfRes.pageCount
|
||||
}
|
||||
} else {
|
||||
await transcodeService.generateSprite(
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
params.mediaInfo.duration,
|
||||
)
|
||||
}
|
||||
|
||||
const spriteBuffer = fs.readFileSync(spriteFile)
|
||||
await s3Service.putObject(
|
||||
@@ -452,6 +492,56 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
}
|
||||
}
|
||||
|
||||
export interface GeneratePdfProxyActivityParams {
|
||||
assetId: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
mediaType: string
|
||||
filename: string
|
||||
}
|
||||
|
||||
export async function generatePdfProxyActivity(
|
||||
params: GeneratePdfProxyActivityParams,
|
||||
): Promise<{ pdfProxyKey: string; pdfFilePath: string }> {
|
||||
const isTxt =
|
||||
params.mediaType === 'text/plain' ||
|
||||
params.mediaType === 'text/markdown' ||
|
||||
params.mediaType === 'text/x-markdown' ||
|
||||
params.filename.toLowerCase().endsWith('.txt') ||
|
||||
params.filename.toLowerCase().endsWith('.md') ||
|
||||
params.filename.toLowerCase().endsWith('.markdown')
|
||||
const isCsv = params.mediaType === 'text/csv' || params.filename.toLowerCase().endsWith('.csv')
|
||||
|
||||
if (!isTxt && !isCsv) {
|
||||
return { pdfProxyKey: params.assetKey, pdfFilePath: params.filePath }
|
||||
}
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const pdfProxyKey = `files/${params.assetId}/proxy.pdf`
|
||||
const tmpDir = path.dirname(params.filePath)
|
||||
const pdfFilePath = path.join(tmpDir, 'proxy.pdf')
|
||||
|
||||
try {
|
||||
if (isCsv) {
|
||||
await transcodeService.generatePdfFromCsv(params.filePath, pdfFilePath)
|
||||
} else {
|
||||
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
|
||||
}
|
||||
|
||||
const buffer = fs.readFileSync(pdfFilePath)
|
||||
await s3Service.putObject(bucket, pdfProxyKey, buffer, buffer.length, 'application/pdf')
|
||||
|
||||
return { pdfProxyKey, pdfFilePath }
|
||||
} catch (err) {
|
||||
const { message } = getErrorDetails(err)
|
||||
throw ApplicationFailure.create({
|
||||
message: `Failed to generate PDF proxy for text/csv/markdown: ${message}`,
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadMediaToTmpActivity(params: {
|
||||
assetKey: string
|
||||
}): Promise<{ filePath: string; tmpDir: string }> {
|
||||
@@ -573,6 +663,36 @@ export async function overlayAnnotationsActivity(params: {
|
||||
}
|
||||
}
|
||||
|
||||
export async function renderPdfPagesActivity(params: {
|
||||
assetKey: string
|
||||
assetId: string
|
||||
start: number
|
||||
end: number
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
}): Promise<Array<{ key: string; page: number }>> {
|
||||
try {
|
||||
return await transcodeService.renderPdfPages(params)
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
lowerMsg.includes('nosuchkey') ||
|
||||
lowerMsg.includes('pdftoppm') ||
|
||||
lowerMsg.includes('sharp')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `PDF page rendering failed: ${message}`,
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateEmbeddingTaskIfEnabledParams {
|
||||
assetId: string
|
||||
teamId: string | null
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
import { WorkflowTaskType } from '@shumai/db'
|
||||
import { registerActivities, registerWorkflow } from '@shumai/workflow-core'
|
||||
import * as transcodeActivities from './activities/transcode'
|
||||
import { transcodeVideoWorkflow } from './workflows/transcode-video'
|
||||
import { transcodeImageWorkflow } from './workflows/transcode-image'
|
||||
import { transcodePdfWorkflow } from './workflows/transcode-pdf'
|
||||
import { renderPdfPagesWorkflow } from './workflows/render-pdf-pages'
|
||||
import { takeVideoScreenshotsWorkflow } from './workflows/take-video-screenshots'
|
||||
import { overlayImageAnnotationWorkflow } from './workflows/overlay-image-annotation'
|
||||
import { transcodeMedia } from './workflows/transcode'
|
||||
|
||||
export function initTranscodeWorkflows() {
|
||||
registerWorkflow(WorkflowTaskType.transcode, transcodeMedia)
|
||||
registerWorkflow(WorkflowTaskType.transcode_video, transcodeVideoWorkflow)
|
||||
registerWorkflow(WorkflowTaskType.transcode_image, transcodeImageWorkflow)
|
||||
registerWorkflow(WorkflowTaskType.transcode_pdf, transcodePdfWorkflow)
|
||||
registerWorkflow(WorkflowTaskType.transcode_pdf_pages, renderPdfPagesWorkflow)
|
||||
registerWorkflow(WorkflowTaskType.transcode_screenshot, takeVideoScreenshotsWorkflow)
|
||||
registerWorkflow(WorkflowTaskType.transcode_image_annotation, overlayImageAnnotationWorkflow)
|
||||
registerActivities(transcodeActivities)
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ export class VideoTranscoder {
|
||||
assetId: this.assetId,
|
||||
teamId: this.teamId,
|
||||
projectId: this.projectId,
|
||||
type: WorkflowTaskType.transcode,
|
||||
type: WorkflowTaskType.transcode_video,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: this.projectId,
|
||||
@@ -71,7 +71,45 @@ export class ImageTranscoder {
|
||||
assetId: this.assetId,
|
||||
teamId: this.teamId,
|
||||
projectId: this.projectId,
|
||||
type: WorkflowTaskType.transcode,
|
||||
type: WorkflowTaskType.transcode_image,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: this.projectId,
|
||||
transcode: this.spec,
|
||||
},
|
||||
},
|
||||
})
|
||||
return task.id
|
||||
}
|
||||
}
|
||||
|
||||
export class PdfTranscoder {
|
||||
private spec: PrismaJson.TaskSpec = {}
|
||||
|
||||
constructor(
|
||||
private readonly db: TransactionClient,
|
||||
private readonly assetId: string,
|
||||
private readonly teamId: string,
|
||||
private readonly projectId: string,
|
||||
) {}
|
||||
|
||||
withSprite(): this {
|
||||
this.spec.sprite = true
|
||||
return this
|
||||
}
|
||||
|
||||
withPoster(): this {
|
||||
this.spec.poster = true
|
||||
return this
|
||||
}
|
||||
|
||||
async submit(): Promise<string> {
|
||||
const task = await this.db.workflowTask.create({
|
||||
data: {
|
||||
assetId: this.assetId,
|
||||
teamId: this.teamId,
|
||||
projectId: this.projectId,
|
||||
type: WorkflowTaskType.transcode_pdf,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: this.projectId,
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import { executeActivity, getActivities, TaskQueueTranscode } from '@shumai/workflow-core'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
|
||||
export async function getWorkerQueueAndStartTask(task: WorkflowTask): Promise<string> {
|
||||
const { getTranscodeWorkerQueueActivity, updateTaskStatusActivity } = getActivities()
|
||||
const workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity)
|
||||
await executeActivity(workerQueue, updateTaskStatusActivity, {
|
||||
taskId: task.id,
|
||||
status: 'processing',
|
||||
})
|
||||
return workerQueue
|
||||
}
|
||||
|
||||
export async function fetchAssetWithKey(workerQueue: string, assetId: string) {
|
||||
const { getAssetActivity } = getActivities()
|
||||
const asset = await executeActivity(workerQueue, getAssetActivity, assetId)
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset not found or has no key',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
return { asset, key }
|
||||
}
|
||||
|
||||
export async function completeTask(
|
||||
workerQueue: string,
|
||||
taskId: string,
|
||||
output?: Record<string, unknown>,
|
||||
) {
|
||||
const { updateTaskStatusActivity } = getActivities()
|
||||
await executeActivity(workerQueue, updateTaskStatusActivity, {
|
||||
taskId,
|
||||
status: 'completed',
|
||||
...(output ? { output } : {}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function failTask(workerQueue: string, taskId: string, err: unknown) {
|
||||
const { updateTaskStatusActivity } = getActivities()
|
||||
if (workerQueue) {
|
||||
await executeActivity(workerQueue, updateTaskStatusActivity, {
|
||||
taskId,
|
||||
status: 'failed',
|
||||
output: { error: err instanceof Error ? err.message : String(err) },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export async function cleanupTmpDir(workerQueue: string, tmpDir: string | undefined) {
|
||||
if (tmpDir && workerQueue) {
|
||||
const { cleanupTmpDirActivity } = getActivities()
|
||||
try {
|
||||
await executeActivity(workerQueue, cleanupTmpDirActivity, { tmpDir })
|
||||
} catch (cleanupErr) {
|
||||
console.error('Failed to cleanup tmp dir:', cleanupErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { overlayImageAnnotationWorkflow } from './overlay-image-annotation'
|
||||
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
import * as workflowUtils from '@shumai/workflow-core'
|
||||
|
||||
vi.mock('@shumai/workflow-core', async (importOriginal) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
getActivities: vi.fn(),
|
||||
executeActivity: vi.fn(),
|
||||
sleep: vi.fn(),
|
||||
}
|
||||
})
|
||||
|
||||
describe('overlayImageAnnotationWorkflow', () => {
|
||||
const mockActivities = {
|
||||
updateTaskStatusActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'updateTaskStatusActivity',
|
||||
}),
|
||||
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
|
||||
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getTranscodeWorkerQueueActivity',
|
||||
}),
|
||||
overlayAnnotationsActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'overlayAnnotationsActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(workflowUtils.executeActivity as any).mockImplementation(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(_queue: string, fn: any, ...args: any[]) => {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
|
||||
}
|
||||
return fn(...args)
|
||||
},
|
||||
)
|
||||
|
||||
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
|
||||
})
|
||||
|
||||
it('should run image annotation overlay workflow successfully', async () => {
|
||||
const task: WorkflowTask = {
|
||||
id: 'task-image-ann',
|
||||
assetId: 'asset-image',
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
sessionId: null,
|
||||
output: null,
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
imageAnnotation: {
|
||||
annotations: [{ type: 'box', color: '#ff0000', points: [] }],
|
||||
},
|
||||
},
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
heartbeat: null,
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
uid: 'task-uid',
|
||||
model: null,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-image',
|
||||
storageKey: { key: 'image.png' },
|
||||
mediaType: 'image/png',
|
||||
media: {
|
||||
imageTranscodes: [{ key: 'image-transcoded.webp' }],
|
||||
},
|
||||
})
|
||||
|
||||
mockActivities.overlayAnnotationsActivity.mockResolvedValue(
|
||||
'files/asset-image/annotations/ann1.webp',
|
||||
)
|
||||
|
||||
await overlayImageAnnotationWorkflow(task)
|
||||
|
||||
expect(mockActivities.overlayAnnotationsActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'image-transcoded.webp',
|
||||
assetId: 'asset-image',
|
||||
annotations: [{ type: 'box', color: '#ff0000', points: [] }],
|
||||
})
|
||||
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-image-ann',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: { key: 'files/asset-image/annotations/ann1.webp' },
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import '@shumai/db/src/prisma-json-types'
|
||||
import { executeActivity, getActivities } from '@shumai/workflow-core'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import { getWorkerQueueAndStartTask, completeTask, failTask } from './common'
|
||||
|
||||
export async function overlayImageAnnotationWorkflow(task: WorkflowTask): Promise<void> {
|
||||
let workerQueue = ''
|
||||
try {
|
||||
workerQueue = await getWorkerQueueAndStartTask(task)
|
||||
|
||||
const payload = task.payload
|
||||
if (!payload?.imageAnnotation) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Invalid payload for imageAnnotation task',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const { getAssetActivity, overlayAnnotationsActivity } = getActivities()
|
||||
const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId)
|
||||
let key = asset?.storageKey?.key
|
||||
if (asset?.media) {
|
||||
const mediaInfo = asset.media as unknown as PrismaJson.MediaInfo
|
||||
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
key = mediaInfo.imageTranscodes[0].key || key
|
||||
}
|
||||
}
|
||||
|
||||
if (!asset || !key) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset not found or has no key',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const keyWithOverlay = await executeActivity(workerQueue, overlayAnnotationsActivity, {
|
||||
assetKey: key,
|
||||
assetId: asset.id,
|
||||
annotations: payload.imageAnnotation.annotations,
|
||||
})
|
||||
|
||||
await completeTask(workerQueue, task.id, { key: keyWithOverlay })
|
||||
} catch (err) {
|
||||
console.error(`overlayImageAnnotationWorkflow failed for task ${task.id}:`, err)
|
||||
await failTask(workerQueue, task.id, err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { renderPdfPagesWorkflow } from './render-pdf-pages'
|
||||
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
import * as workflowUtils from '@shumai/workflow-core'
|
||||
|
||||
vi.mock('@shumai/workflow-core', async (importOriginal) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
getActivities: vi.fn(),
|
||||
executeActivity: vi.fn(),
|
||||
sleep: vi.fn(),
|
||||
}
|
||||
})
|
||||
|
||||
describe('renderPdfPagesWorkflow', () => {
|
||||
const mockActivities = {
|
||||
updateTaskStatusActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'updateTaskStatusActivity',
|
||||
}),
|
||||
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
|
||||
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getTranscodeWorkerQueueActivity',
|
||||
}),
|
||||
renderPdfPagesActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'renderPdfPagesActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(workflowUtils.executeActivity as any).mockImplementation(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(_queue: string, fn: any, ...args: any[]) => {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
|
||||
}
|
||||
return fn(...args)
|
||||
},
|
||||
)
|
||||
|
||||
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
|
||||
})
|
||||
|
||||
it('should run renderPdfPagesWorkflow successfully', async () => {
|
||||
const task: WorkflowTask = {
|
||||
id: 'task-pdf-pages',
|
||||
assetId: 'asset-pdf',
|
||||
type: WorkflowTaskType.transcode_pdf_pages,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
pdfPages: {
|
||||
start: 1,
|
||||
end: 2,
|
||||
commentTimestamp: 2,
|
||||
annotations: [],
|
||||
},
|
||||
},
|
||||
output: null,
|
||||
sessionId: null,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
heartbeat: null,
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
uid: 'task-uid',
|
||||
model: null,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-pdf',
|
||||
storageKey: { key: 'doc.pdf' },
|
||||
mediaType: 'application/pdf',
|
||||
})
|
||||
|
||||
mockActivities.renderPdfPagesActivity.mockResolvedValue([
|
||||
{ key: 'pdf_pages/doc-page-1.webp', page: 1 },
|
||||
{ key: 'pdf_pages/doc-page-2.webp', page: 2 },
|
||||
])
|
||||
|
||||
await renderPdfPagesWorkflow(task)
|
||||
|
||||
expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'doc.pdf',
|
||||
assetId: 'asset-pdf',
|
||||
start: 1,
|
||||
end: 2,
|
||||
commentTimestamp: 2,
|
||||
annotations: [],
|
||||
})
|
||||
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-pdf-pages',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
pages: [
|
||||
{ key: 'pdf_pages/doc-page-1.webp', page: 1 },
|
||||
{ key: 'pdf_pages/doc-page-2.webp', page: 2 },
|
||||
],
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import { executeActivity, getActivities } from '@shumai/workflow-core'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import { getWorkerQueueAndStartTask, fetchAssetWithKey, completeTask, failTask } from './common'
|
||||
|
||||
export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void> {
|
||||
let workerQueue = ''
|
||||
try {
|
||||
workerQueue = await getWorkerQueueAndStartTask(task)
|
||||
|
||||
const payload = task.payload
|
||||
if (!payload?.pdfPages) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Invalid payload for pdfPages task',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
|
||||
const { renderPdfPagesActivity } = getActivities()
|
||||
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
|
||||
|
||||
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
|
||||
assetKey: pdfKey,
|
||||
assetId: asset.id,
|
||||
start: payload.pdfPages.start,
|
||||
end: payload.pdfPages.end,
|
||||
commentTimestamp: payload.pdfPages.commentTimestamp,
|
||||
annotations: payload.pdfPages.annotations,
|
||||
})
|
||||
|
||||
await completeTask(workerQueue, task.id, { pages })
|
||||
} catch (err) {
|
||||
console.error(`renderPdfPagesWorkflow failed for task ${task.id}:`, err)
|
||||
await failTask(workerQueue, task.id, err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { takeVideoScreenshotsWorkflow } from './take-video-screenshots'
|
||||
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
import * as workflowUtils from '@shumai/workflow-core'
|
||||
|
||||
vi.mock('@shumai/workflow-core', async (importOriginal) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const actual = (await importOriginal()) as any
|
||||
return {
|
||||
...actual,
|
||||
getActivities: vi.fn(),
|
||||
executeActivity: vi.fn(),
|
||||
sleep: vi.fn(),
|
||||
}
|
||||
})
|
||||
|
||||
describe('takeVideoScreenshotsWorkflow', () => {
|
||||
const mockActivities = {
|
||||
updateTaskStatusActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'updateTaskStatusActivity',
|
||||
}),
|
||||
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
|
||||
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getTranscodeWorkerQueueActivity',
|
||||
}),
|
||||
takeScreenshotsActivity: Object.assign(vi.fn(), { _activityName: 'takeScreenshotsActivity' }),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(workflowUtils.getActivities as any).mockReturnValue(mockActivities)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(workflowUtils.executeActivity as any).mockImplementation(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(_queue: string, fn: any, ...args: any[]) => {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`fn is not a function in executeActivity. Queue: ${_queue}`)
|
||||
}
|
||||
return fn(...args)
|
||||
},
|
||||
)
|
||||
|
||||
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_worker_queue')
|
||||
})
|
||||
|
||||
it('should run screenshot workflow successfully', async () => {
|
||||
const task: WorkflowTask = {
|
||||
id: 'task-screenshot',
|
||||
assetId: 'asset-video',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
sessionId: null,
|
||||
output: null,
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
screenshot: {
|
||||
start: 0,
|
||||
end: 10,
|
||||
count: 2,
|
||||
commentTimestamp: 5.0,
|
||||
annotations: [],
|
||||
},
|
||||
},
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
heartbeat: null,
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
uid: 'task-uid',
|
||||
model: null,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-video',
|
||||
storageKey: { key: 'video.mp4' },
|
||||
mediaType: 'video/mp4',
|
||||
})
|
||||
|
||||
mockActivities.takeScreenshotsActivity.mockResolvedValue([
|
||||
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
|
||||
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
|
||||
])
|
||||
|
||||
await takeVideoScreenshotsWorkflow(task)
|
||||
|
||||
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'video.mp4',
|
||||
assetId: 'asset-video',
|
||||
start: 0,
|
||||
end: 10,
|
||||
count: 2,
|
||||
commentTimestamp: 5.0,
|
||||
annotations: [],
|
||||
})
|
||||
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-screenshot',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
screenshots: [
|
||||
{ key: 'files/asset-video/screenshots/shot1.webp', timestamp: 0.0 },
|
||||
{ key: 'files/asset-video/screenshots/shot2.webp', timestamp: 5.0 },
|
||||
],
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import { executeActivity, getActivities } from '@shumai/workflow-core'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import { getWorkerQueueAndStartTask, fetchAssetWithKey, completeTask, failTask } from './common'
|
||||
|
||||
export async function takeVideoScreenshotsWorkflow(task: WorkflowTask): Promise<void> {
|
||||
let workerQueue = ''
|
||||
try {
|
||||
workerQueue = await getWorkerQueueAndStartTask(task)
|
||||
|
||||
const payload = task.payload
|
||||
if (!payload?.screenshot) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Invalid payload for screenshot task',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
|
||||
const { takeScreenshotsActivity } = getActivities()
|
||||
|
||||
const screenshots = await executeActivity(workerQueue, takeScreenshotsActivity, {
|
||||
assetKey: key,
|
||||
assetId: asset.id,
|
||||
start: payload.screenshot.start,
|
||||
end: payload.screenshot.end,
|
||||
count: payload.screenshot.count,
|
||||
commentTimestamp: payload.screenshot.commentTimestamp,
|
||||
annotations: payload.screenshot.annotations,
|
||||
})
|
||||
|
||||
await completeTask(workerQueue, task.id, { screenshots })
|
||||
} catch (err) {
|
||||
console.error(`takeVideoScreenshotsWorkflow failed for task ${task.id}:`, err)
|
||||
await failTask(workerQueue, task.id, err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user