mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-20 11:28:21 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62e86b703e | ||
|
|
6c87db0581 | ||
|
|
bc85dc4541 | ||
|
|
659a0af2c5 | ||
|
|
54d2c407b5 | ||
|
|
8e31d05f8a | ||
|
|
8c1e59d131 | ||
|
|
b087bf73ae | ||
|
|
a49722d5c7 | ||
|
|
576d8ebe64 | ||
|
|
ffc1889cdc | ||
|
|
6285ff8618 | ||
|
|
1e60933cdb | ||
|
|
dd0ea05a88 | ||
|
|
52330869a2 | ||
|
|
0c561e9b39 | ||
|
|
a236cd3a97 | ||
|
|
c3ef769f51 | ||
|
|
e58dfaa71a | ||
|
|
35b57cc0ab | ||
|
|
5a754bcb94 | ||
|
|
64b68e7d63 | ||
|
|
c2f477a391 | ||
|
|
9263971653 | ||
|
|
79a479c87c | ||
|
|
134eb8b7fa | ||
|
|
e06478df88 |
+2
-1
@@ -14,4 +14,5 @@ helm-charts
|
||||
.env
|
||||
.editorconfig
|
||||
.idea
|
||||
coverage*
|
||||
coverage*
|
||||
packages/webui/paraglide
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.14
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
@@ -34,6 +34,9 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils imagemagick
|
||||
|
||||
- name: Compile i18n
|
||||
run: bun run i18n:compile
|
||||
|
||||
- name: Lint
|
||||
run: bun run lint
|
||||
|
||||
@@ -78,11 +81,14 @@ jobs:
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.14
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
|
||||
- name: Compile i18n
|
||||
run: bun run i18n:compile
|
||||
|
||||
- name: Install Playwright browser (${{ matrix.project }})
|
||||
run: bunx playwright install --with-deps ${{ matrix.project }}
|
||||
|
||||
@@ -120,7 +126,7 @@ jobs:
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.14
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
@@ -130,6 +136,9 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep bubblewrap socat ffmpeg poppler-utils imagemagick
|
||||
|
||||
- name: Compile i18n
|
||||
run: bun run i18n:compile
|
||||
|
||||
- name: Install Playwright browser (${{ matrix.project }})
|
||||
run: bunx playwright install --with-deps ${{ matrix.project }}
|
||||
|
||||
@@ -154,7 +163,7 @@ jobs:
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.14
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.3.14
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -53,6 +53,9 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: bun install
|
||||
|
||||
- name: Compile i18n
|
||||
run: bun run i18n:compile
|
||||
|
||||
- name: Lint
|
||||
run: bun run lint
|
||||
|
||||
|
||||
@@ -19,3 +19,7 @@ code_quality.md
|
||||
/test-results/
|
||||
/blob-report/
|
||||
/playwright/.cache/
|
||||
|
||||
# Paraglide JS
|
||||
packages/webui/paraglide
|
||||
|
||||
|
||||
+5
-2
@@ -1,6 +1,6 @@
|
||||
#syntax=docker/dockerfile:1.7-labs
|
||||
# Stage 1: Builder
|
||||
FROM oven/bun:1.3.14 AS builder
|
||||
FROM oven/bun:1.4.2 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -18,6 +18,9 @@ RUN ls -la packages/transcode/node_modules
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Compile i18n
|
||||
RUN bun run i18n:compile
|
||||
|
||||
# Build API and Worker
|
||||
RUN bun run build.ts
|
||||
|
||||
@@ -25,7 +28,7 @@ RUN bun run build.ts
|
||||
RUN bun run scripts/extract-runtime-deps.ts
|
||||
|
||||
# Stage 2: Runner
|
||||
FROM oven/bun:1.3.14 AS runner
|
||||
FROM oven/bun:1.4.2 AS runner
|
||||
|
||||
# Install ffmpeg/ffprobe
|
||||
COPY --from=mwader/static-ffmpeg:8.1 /ffmpeg /usr/local/bin/
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
<a href="https://hub.docker.com/r/shumaione/shumai"><img alt="Docker pulls" src="https://img.shields.io/docker/pulls/shumaione/shumai" /></a>
|
||||
<a href="https://github.com/shumaiOne/shumai/releases"><img alt="GitHub latest release" src="https://img.shields.io/github/v/release/shumaiOne/shumai" /></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-MIT-blue" /></a>
|
||||
<a target="_blank" href="https://deepwiki.com/shumaiOne/shumai"><img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -314,11 +315,17 @@ To set up Shumai locally for development:
|
||||
```
|
||||
2. Start the `pgvector` database container (as described in **Option 2, Step 1**).
|
||||
3. Create a `.env` file at the root of the workspace using the configuration from **Option 2, Step 5**.
|
||||
4. Apply the database schema migrations:
|
||||
4. Generate i18n translation files:
|
||||
```bash
|
||||
bun run i18n:compile
|
||||
```
|
||||
> [!TIP]
|
||||
> During frontend development, you can run `bun run i18n:watch` in a separate terminal to automatically recompile translation files whenever you edit strings in `packages/webui/messages/`.
|
||||
5. Apply the database schema migrations:
|
||||
```bash
|
||||
bun run db:migrate
|
||||
```
|
||||
5. Start the local development server:
|
||||
6. Start the local development server:
|
||||
```bash
|
||||
bun run dev
|
||||
```
|
||||
|
||||
+28
-6
@@ -12,6 +12,7 @@
|
||||
<a href="https://hub.docker.com/r/shumaione/shumai"><img alt="Docker pulls" src="https://img.shields.io/docker/pulls/shumaione/shumai" /></a>
|
||||
<a href="https://github.com/shumaiOne/shumai/releases"><img alt="GitHub latest release" src="https://img.shields.io/github/v/release/shumaiOne/shumai" /></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-MIT-blue" /></a>
|
||||
<a target="_blank" href="https://deepwiki.com/shumaiOne/shumai"><img alt="Ask DeepWiki" src="https://deepwiki.com/badge.svg" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -69,6 +70,24 @@ Shumai 也将项目管理融入了创意工作流程。 内置的看板可以帮
|
||||
以下是使用本地存储运行 Shumai 的快速上手指南。如需了解高级配置(包括 S3 兼容存储配置、Temporal 工作流编排等),请参阅我们的[官方文档](https://docs.shumai.one/getting-started/overview)。
|
||||
|
||||
### 方案一:使用 Docker Compose(推荐)
|
||||
<details>
|
||||
|
||||
<summary>复制这个提示词,让agent帮你安装</summary>
|
||||
|
||||
```
|
||||
请帮我在本机使用 Docker Compose 一键部署 Shumai。
|
||||
|
||||
请严格参考官方文档:
|
||||
https://gh-proxy.com/https://raw.githubusercontent.com/shumaiOne/shumai/main/README_CN.md
|
||||
|
||||
要求:
|
||||
|
||||
1. 只能使用 Docker Compose local方式部署,不要使用其他部署方式。
|
||||
2. 部署前请先检查本机是否已安装并可正常使用 Docker 和 Docker Compose。
|
||||
3. 如果本机未安装 Docker,请立即提醒用户先安装 Docker,并停止后续部署操作。不要自行安装 Docker,也不要采用其他方式部署。
|
||||
4. 获取 `docker-compose-aliyun.yaml` 文件内容时,必须通过 `https://gh-proxy.com` 加速访问,不要直接访问 GitHub Raw 地址。
|
||||
```
|
||||
</details>
|
||||
|
||||
使用 Docker Compose 是运行 Shumai 最快的方式。您无需克隆仓库,也不需要手动安装依赖包。请确保电脑上已安装 [Docker](https://docs.docker.com/get-docker/) 和 [Docker Compose](https://docs.docker.com/compose/install/),然后按照以下步骤操作:
|
||||
|
||||
@@ -339,17 +358,20 @@ bun install
|
||||
|
||||
2. 启动 `pgvector` 数据库容器(参考**方案二中的第一步**)。
|
||||
3. 参照**方案二中的第五步**,在项目根目录下创建一个 `.env` 文件。
|
||||
4. 执行数据库结构迁移:
|
||||
4. 生成多语言国际化(i18n)代码:
|
||||
```bash
|
||||
bun run i18n:compile
|
||||
```
|
||||
> [!TIP]
|
||||
> 在前端开发过程中,可以在单独的终端窗口中运行 `bun run i18n:watch`,在修改 `packages/webui/messages/` 下的翻译文案时自动重新编译。
|
||||
|
||||
5. 执行数据库结构迁移:
|
||||
```bash
|
||||
bun run db:migrate
|
||||
|
||||
```
|
||||
|
||||
|
||||
5. 启动本地开发服务器:
|
||||
6. 启动本地开发服务器:
|
||||
```bash
|
||||
bun run dev
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -20,6 +20,9 @@ export SHUMAI_API_SERVER="http://localhost:3000"
|
||||
|
||||
# Your personal API Token (Generate this in the WebUI via Settings -> Developer tab)
|
||||
export SHUMAI_API_KEY="your-developer-api-token"
|
||||
|
||||
# Optional: Enable asset deletion capability (disabled by default for safety)
|
||||
export SHUMAI_ALLOW_DELETE="true"
|
||||
```
|
||||
|
||||
---
|
||||
@@ -89,6 +92,37 @@ shumai-cli create-version <localFilePath> -p <parentAssetId>
|
||||
- **Options:**
|
||||
- `-p`, `--parent <id>`: The parent asset ID (file or version stack) to version (required).
|
||||
|
||||
#### 6. Rename an Asset
|
||||
|
||||
Rename an existing file or folder in Shumai.
|
||||
|
||||
```bash
|
||||
shumai-cli rename <assetId> <newName>
|
||||
```
|
||||
|
||||
#### 7. Move Assets
|
||||
|
||||
Move one or more assets (files or folders) to a destination parent folder. Note that all assets to be moved must belong to the same current parent folder.
|
||||
|
||||
```bash
|
||||
shumai-cli move <assetId...> -p <parentId>
|
||||
# or using the short alias:
|
||||
shumai-cli mv <assetId...> -p <parentId>
|
||||
```
|
||||
|
||||
- **Options:**
|
||||
- `-p`, `--parent <id>`: The destination parent folder ID (required).
|
||||
|
||||
#### 8. Delete an Asset
|
||||
|
||||
Delete a single file or folder by moving it to the Trash bin. For safety, this command only deletes one asset at a time and requires the `SHUMAI_ALLOW_DELETE="true"` environment variable to be set in your environment.
|
||||
|
||||
```bash
|
||||
shumai-cli delete <assetId>
|
||||
# or using the short alias:
|
||||
shumai-cli rm <assetId>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
@@ -11,7 +11,7 @@
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@shumai/tableprinter": "workspace:*",
|
||||
"hono": "4.12.23",
|
||||
"ulid": "^3.0.2"
|
||||
"hono": "4.13.5",
|
||||
"ulid": "3.0.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,9 @@ import { mkdir } from './mkdir'
|
||||
import { getClient } from '../client'
|
||||
import { upload } from './upload'
|
||||
import { createVersion } from './create-version'
|
||||
import { rename } from './rename'
|
||||
import { move } from './move'
|
||||
import { deleteAsset } from './delete'
|
||||
import fs from 'node:fs'
|
||||
import http from 'node:http'
|
||||
|
||||
@@ -34,17 +37,29 @@ describe('CLI Commands', () => {
|
||||
$get: vi.fn(),
|
||||
':projectId': {
|
||||
$get: vi.fn(),
|
||||
reparent: {
|
||||
$post: vi.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
folders: {
|
||||
$post: vi.fn(),
|
||||
$delete: vi.fn(),
|
||||
':folderId': {
|
||||
$get: vi.fn(),
|
||||
$put: vi.fn(),
|
||||
children: {
|
||||
$get: vi.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
files: {
|
||||
$delete: vi.fn(),
|
||||
':fileId': {
|
||||
$get: vi.fn(),
|
||||
$put: vi.fn(),
|
||||
},
|
||||
},
|
||||
teams: {
|
||||
':teamId': {
|
||||
upload: {
|
||||
@@ -60,6 +75,8 @@ describe('CLI Commands', () => {
|
||||
},
|
||||
}
|
||||
|
||||
const originalAllowDelete = process.env.SHUMAI_ALLOW_DELETE
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks()
|
||||
logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
@@ -79,6 +96,11 @@ describe('CLI Commands', () => {
|
||||
writeSpy.mockRestore()
|
||||
errorSpy.mockRestore()
|
||||
exitSpy.mockRestore()
|
||||
if (originalAllowDelete !== undefined) {
|
||||
process.env.SHUMAI_ALLOW_DELETE = originalAllowDelete
|
||||
} else {
|
||||
delete process.env.SHUMAI_ALLOW_DELETE
|
||||
}
|
||||
})
|
||||
|
||||
describe('project ls', () => {
|
||||
@@ -361,4 +383,192 @@ describe('CLI Commands', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('rename', () => {
|
||||
it('exits with error when assetId or newName is missing', async () => {
|
||||
await expect(rename('', 'new-name')).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Asset ID is required'))
|
||||
|
||||
await expect(rename('asset-1', '')).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('New name is required'))
|
||||
})
|
||||
|
||||
it('renames a folder when asset type is folder', async () => {
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'f-1', type: 'folder' }),
|
||||
} as unknown as Response)
|
||||
mockClient.api.folders[':folderId'].$put.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'f-1', name: 'Renamed Folder' }),
|
||||
} as unknown as Response)
|
||||
|
||||
await rename('f-1', 'Renamed Folder')
|
||||
|
||||
expect(mockClient.api.folders[':folderId'].$put).toHaveBeenCalledWith({
|
||||
param: { folderId: 'f-1' },
|
||||
json: { name: 'Renamed Folder' },
|
||||
})
|
||||
expect(logSpy).toHaveBeenCalledWith('Renamed asset f-1 to "Renamed Folder"')
|
||||
})
|
||||
|
||||
it('renames a file when asset type is file', async () => {
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'file-1', type: 'file' }),
|
||||
} as unknown as Response)
|
||||
mockClient.api.files[':fileId'].$put.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'file-1', name: 'new.mp4' }),
|
||||
} as unknown as Response)
|
||||
|
||||
await rename('file-1', 'new.mp4')
|
||||
|
||||
expect(mockClient.api.files[':fileId'].$put).toHaveBeenCalledWith({
|
||||
param: { fileId: 'file-1' },
|
||||
json: { name: 'new.mp4' },
|
||||
})
|
||||
expect(logSpy).toHaveBeenCalledWith('Renamed asset file-1 to "new.mp4"')
|
||||
})
|
||||
|
||||
it('exits with error when asset fetch fails', async () => {
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: false,
|
||||
json: async () => ({ error: 'Asset not found' }),
|
||||
} as unknown as Response)
|
||||
|
||||
await expect(rename('unknown-id', 'name')).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith('Error: Asset not found')
|
||||
})
|
||||
})
|
||||
|
||||
describe('move', () => {
|
||||
it('exits with error when assetIds or parentId is missing', async () => {
|
||||
await expect(move([], 'parent-1')).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('At least one asset ID is required'),
|
||||
)
|
||||
|
||||
await expect(move(['a-1'], '')).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Option -p/--parent <parentId> is required'),
|
||||
)
|
||||
})
|
||||
|
||||
it('successfully moves assets', async () => {
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'parent-1', projectId: 'proj-1' }),
|
||||
} as unknown as Response)
|
||||
mockClient.api.projects[':projectId'].reparent.$post.mockResolvedValue({
|
||||
ok: true,
|
||||
} as unknown as Response)
|
||||
|
||||
await move(['a-1', 'a-2'], 'parent-1')
|
||||
|
||||
expect(mockClient.api.projects[':projectId'].reparent.$post).toHaveBeenCalledWith({
|
||||
param: { projectId: 'proj-1' },
|
||||
json: {
|
||||
assetIds: ['a-1', 'a-2'],
|
||||
newParentId: 'parent-1',
|
||||
},
|
||||
})
|
||||
expect(logSpy).toHaveBeenCalledWith('Moved 2 asset(s) to parent parent-1')
|
||||
})
|
||||
|
||||
it('exits with error when destination parent is not found', async () => {
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: false,
|
||||
json: async () => ({ error: 'Folder not found' }),
|
||||
} as unknown as Response)
|
||||
|
||||
await expect(move(['a-1'], 'missing-parent')).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith('Error: Folder not found')
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteAsset', () => {
|
||||
it('exits with error when SHUMAI_ALLOW_DELETE is not set or not true', async () => {
|
||||
delete process.env.SHUMAI_ALLOW_DELETE
|
||||
await expect(deleteAsset(['a-1'])).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
'Error: Asset deletion is disabled in this environment.',
|
||||
)
|
||||
})
|
||||
|
||||
it('exits with error when assetIds is empty', async () => {
|
||||
process.env.SHUMAI_ALLOW_DELETE = 'true'
|
||||
await expect(deleteAsset([])).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Asset ID is required'))
|
||||
})
|
||||
|
||||
it('exits with error when more than one asset ID is passed', async () => {
|
||||
process.env.SHUMAI_ALLOW_DELETE = 'true'
|
||||
await expect(deleteAsset(['a-1', 'a-2'])).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith('Error: Can only delete one asset at a time.')
|
||||
})
|
||||
|
||||
it('exits with error when asset is not found', async () => {
|
||||
process.env.SHUMAI_ALLOW_DELETE = 'true'
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: false,
|
||||
json: async () => ({ error: 'Asset not found' }),
|
||||
} as unknown as Response)
|
||||
|
||||
await expect(deleteAsset(['a-1'])).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith('Error: Asset not found')
|
||||
})
|
||||
|
||||
it('successfully deletes a file when SHUMAI_ALLOW_DELETE is true', async () => {
|
||||
process.env.SHUMAI_ALLOW_DELETE = 'true'
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'a-1', type: 'file' }),
|
||||
} as unknown as Response)
|
||||
mockClient.api.files.$delete.mockResolvedValue({
|
||||
ok: true,
|
||||
} as unknown as Response)
|
||||
|
||||
await deleteAsset(['a-1'])
|
||||
|
||||
expect(mockClient.api.files.$delete).toHaveBeenCalledWith({
|
||||
json: { ids: ['a-1'] },
|
||||
})
|
||||
expect(mockClient.api.folders.$delete).not.toHaveBeenCalled()
|
||||
expect(logSpy).toHaveBeenCalledWith('Deleted asset a-1')
|
||||
})
|
||||
|
||||
it('successfully deletes a folder when SHUMAI_ALLOW_DELETE is 1', async () => {
|
||||
process.env.SHUMAI_ALLOW_DELETE = '1'
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'folder-1', type: 'folder' }),
|
||||
} as unknown as Response)
|
||||
mockClient.api.folders.$delete.mockResolvedValue({
|
||||
ok: true,
|
||||
} as unknown as Response)
|
||||
|
||||
await deleteAsset(['folder-1'])
|
||||
|
||||
expect(mockClient.api.folders.$delete).toHaveBeenCalledWith({
|
||||
json: { ids: ['folder-1'] },
|
||||
})
|
||||
expect(logSpy).toHaveBeenCalledWith('Deleted asset folder-1')
|
||||
})
|
||||
|
||||
it('exits with error when delete API returns an error', async () => {
|
||||
process.env.SHUMAI_ALLOW_DELETE = 'true'
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'a-1', type: 'file' }),
|
||||
} as unknown as Response)
|
||||
mockClient.api.files.$delete.mockResolvedValue({
|
||||
ok: false,
|
||||
json: async () => ({ error: 'Failed to delete' }),
|
||||
} as unknown as Response)
|
||||
|
||||
await expect(deleteAsset(['a-1'])).rejects.toThrow('process.exit(1)')
|
||||
expect(errorSpy).toHaveBeenCalledWith('Error: Failed to delete')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { getClient } from '../client'
|
||||
|
||||
export async function deleteAsset(assetIds: string[]) {
|
||||
const isAllowed =
|
||||
process.env.SHUMAI_ALLOW_DELETE === 'true' || process.env.SHUMAI_ALLOW_DELETE === '1'
|
||||
if (!isAllowed) {
|
||||
console.error('Error: Asset deletion is disabled in this environment.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (!assetIds || assetIds.length === 0) {
|
||||
console.error('Error: Asset ID is required. Usage: shumai-cli delete <assetId>')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (assetIds.length > 1) {
|
||||
console.error('Error: Can only delete one asset at a time.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const assetId = assetIds[0]
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
// Inspect asset to determine whether it is a folder or file
|
||||
const getRes = await client.api.folders[':folderId'].$get({
|
||||
param: { folderId: assetId },
|
||||
})
|
||||
|
||||
if (!getRes.ok) {
|
||||
const err = (await getRes.json().catch(() => ({ error: 'Asset not found' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Asset not found'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const asset = await getRes.json()
|
||||
const isFolder = asset.type === 'folder'
|
||||
|
||||
const res = isFolder
|
||||
? await client.api.folders.$delete({
|
||||
json: { ids: [assetId] },
|
||||
})
|
||||
: await client.api.files.$delete({
|
||||
json: { ids: [assetId] },
|
||||
})
|
||||
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({ error: 'Failed to delete asset' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Failed to delete asset'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log(`Deleted asset ${assetId}`)
|
||||
} catch (err) {
|
||||
console.error('Error connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { getClient } from '../client'
|
||||
|
||||
export async function move(assetIds: string[], parentId: string) {
|
||||
if (!assetIds || assetIds.length === 0) {
|
||||
console.error(
|
||||
'Error: At least one asset ID is required. Usage: shumai-cli move <assetId...> -p <parentId>',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Option -p/--parent <parentId> is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
// Fetch destination folder to get its projectId
|
||||
const parentRes = await client.api.folders[':folderId'].$get({
|
||||
param: { folderId: parentId },
|
||||
})
|
||||
|
||||
if (!parentRes.ok) {
|
||||
const err = (await parentRes.json().catch(() => ({ error: 'Parent folder not found' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Parent folder not found'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const parentInfo = await parentRes.json()
|
||||
const projectId = parentInfo.projectId
|
||||
if (!projectId) {
|
||||
console.error('Error: Destination parent folder is not associated with a project.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const res = await client.api.projects[':projectId'].reparent.$post({
|
||||
param: { projectId },
|
||||
json: {
|
||||
assetIds,
|
||||
newParentId: parentId,
|
||||
},
|
||||
})
|
||||
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({ error: 'Failed to move assets' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Failed to move assets'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log(`Moved ${assetIds.length} asset(s) to parent ${parentId}`)
|
||||
} catch (err) {
|
||||
console.error('Error connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { getClient } from '../client'
|
||||
|
||||
export async function rename(assetId: string, newName: string) {
|
||||
if (!assetId) {
|
||||
console.error('Error: Asset ID is required. Usage: shumai-cli rename <assetId> <newName>')
|
||||
process.exit(1)
|
||||
}
|
||||
if (!newName) {
|
||||
console.error('Error: New name is required. Usage: shumai-cli rename <assetId> <newName>')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
// Determine whether asset is a folder or a file
|
||||
const getRes = await client.api.folders[':folderId'].$get({
|
||||
param: { folderId: assetId },
|
||||
})
|
||||
|
||||
if (!getRes.ok) {
|
||||
const err = (await getRes.json().catch(() => ({ error: 'Asset not found' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Asset not found'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const asset = await getRes.json()
|
||||
const isFolder = asset.type === 'folder'
|
||||
|
||||
const updateRes = isFolder
|
||||
? await client.api.folders[':folderId'].$put({
|
||||
param: { folderId: assetId },
|
||||
json: { name: newName },
|
||||
})
|
||||
: await client.api.files[':fileId'].$put({
|
||||
param: { fileId: assetId },
|
||||
json: { name: newName },
|
||||
})
|
||||
|
||||
if (!updateRes.ok) {
|
||||
const err = (await updateRes.json().catch(() => ({ error: 'Failed to rename asset' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Failed to rename asset'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log(`Renamed asset ${assetId} to "${newName}"`)
|
||||
} catch (err) {
|
||||
console.error('Error connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
+37
-1
@@ -3,6 +3,9 @@ import { ls } from './commands/ls'
|
||||
import { mkdir } from './commands/mkdir'
|
||||
import { upload } from './commands/upload'
|
||||
import { createVersion } from './commands/create-version'
|
||||
import { rename } from './commands/rename'
|
||||
import { move } from './commands/move'
|
||||
import { deleteAsset } from './commands/delete'
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
@@ -14,9 +17,12 @@ Commands:
|
||||
mkdir <name> -p <parentId> Create a new folder in a parent folder
|
||||
upload <path> -p <parentId> Upload a local file or folder to a parent folder
|
||||
create-version <path> -p <id> Create a new version of an existing file or version stack
|
||||
rename <assetId> <newName> Rename an existing file or folder
|
||||
move <assetIds...> -p <target> Move one or more assets to a destination parent folder (alias: mv)
|
||||
delete <assetId> Delete a single file or folder (alias: rm)
|
||||
|
||||
Options:
|
||||
-p, --parent <id> The parent folder/asset ID (required for ls, mkdir, upload, create-version)
|
||||
-p, --parent <id> The parent folder/asset ID (required for ls, mkdir, upload, create-version, move)
|
||||
-h, --help Show help details
|
||||
`)
|
||||
}
|
||||
@@ -92,6 +98,36 @@ async function main() {
|
||||
process.exit(1)
|
||||
}
|
||||
await createVersion(localPath, parentId)
|
||||
} else if (cmd === 'rename') {
|
||||
const assetId = cleanArgs[1]
|
||||
const newName = cleanArgs[2]
|
||||
if (!assetId || !newName) {
|
||||
console.error(
|
||||
'Error: Asset ID and new name are required. Usage: shumai-cli rename <assetId> <newName>',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
await rename(assetId, newName)
|
||||
} else if (cmd === 'move' || cmd === 'mv') {
|
||||
const assetIds = cleanArgs.slice(1)
|
||||
if (assetIds.length === 0) {
|
||||
console.error(
|
||||
'Error: At least one asset ID is required. Usage: shumai-cli move <assetId...> -p <parentId>',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Option -p/--parent <parentId> is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
await move(assetIds, parentId)
|
||||
} else if (cmd === 'delete' || cmd === 'rm') {
|
||||
const assetIds = cleanArgs.slice(1)
|
||||
if (assetIds.length === 0) {
|
||||
console.error('Error: Asset ID is required. Usage: shumai-cli delete <assetId>')
|
||||
process.exit(1)
|
||||
}
|
||||
await deleteAsset(assetIds)
|
||||
} else {
|
||||
console.error(`Error: Unknown command "${cmd}". Run "shumai-cli --help" for usage.`)
|
||||
process.exit(1)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -24,14 +24,32 @@ export async function dragCard(page: Page, source: Locator, target: Locator): Pr
|
||||
await page.mouse.move(fromX, fromY)
|
||||
await page.mouse.down()
|
||||
|
||||
// Move in several chunks so the dnd-kit sensor activates mid-drag
|
||||
const dx = toX - fromX
|
||||
const dy = toY - fromY
|
||||
const dist = Math.hypot(dx, dy)
|
||||
|
||||
// Step past the 10px activation constraint to trigger drag activation
|
||||
// and give the browser a brief moment for status to become 'dragging'
|
||||
const activationRatio = dist > 0 ? Math.min(1, 15 / dist) : 1
|
||||
await page.mouse.move(fromX + dx * activationRatio, fromY + dy * activationRatio)
|
||||
await page.waitForTimeout(50)
|
||||
|
||||
// Move in several chunks so the intermediate dragmove handlers run
|
||||
const chunks = 8
|
||||
for (let i = 1; i <= chunks; i++) {
|
||||
const x = fromX + ((toX - fromX) * i) / chunks
|
||||
const y = fromY + ((toY - fromY) * i) / chunks
|
||||
const x = fromX + (dx * i) / chunks
|
||||
const y = fromY + (dy * i) / chunks
|
||||
await page.mouse.move(x, y)
|
||||
await page.waitForTimeout(15)
|
||||
await page.waitForTimeout(20)
|
||||
}
|
||||
|
||||
// Ensure pointer is directly over target center and allow requestAnimationFrame
|
||||
// and collision microtasks to settle before triggering drop
|
||||
await page.mouse.move(toX, toY)
|
||||
await page.evaluate(
|
||||
() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))),
|
||||
)
|
||||
await page.waitForTimeout(100)
|
||||
|
||||
await page.mouse.up()
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ interface CreateShareResponse {
|
||||
*/
|
||||
export function fileCard(page: Page, name: string): Locator {
|
||||
return page
|
||||
.locator('div.group')
|
||||
.locator('[data-testid="file-card"]')
|
||||
.filter({
|
||||
has: page.locator(`input[value="${name}"]`).or(page.getByText(name, { exact: true })),
|
||||
})
|
||||
|
||||
@@ -12,6 +12,11 @@ test('owner manages resource quotas in settings and resets usage from the dashbo
|
||||
// 2. Click the Quotas sidebar tab
|
||||
await page.getByRole('button', { name: /Quotas|配额/i }).click()
|
||||
|
||||
// Verify URL contains #quotas and remains after reload
|
||||
expect(page.url()).toContain('#quotas')
|
||||
await page.reload()
|
||||
expect(page.url()).toContain('#quotas')
|
||||
|
||||
// 3. Verify Quotas settings card is displayed with empty state
|
||||
await expect(page.getByRole('heading', { name: /^(Resource Quotas|资源配额)$/i })).toBeVisible()
|
||||
await expect(page.getByText(/No resource quotas configured|暂未配置资源配额/i)).toBeVisible()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
@@ -12,11 +12,11 @@
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/transcode": "workspace:*",
|
||||
"@shumai/webui": "workspace:*",
|
||||
"hono": "4.12.23"
|
||||
"hono": "4.13.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@prisma/adapter-pg": "7.8.0",
|
||||
"pg": "8.21.0",
|
||||
"@types/pg": "8.20.0"
|
||||
"@prisma/adapter-pg": "7.10.0",
|
||||
"@types/pg": "8.23.1",
|
||||
"pg": "8.23.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { migrateLegacyAgentAvatars } from '@shumai/core/src/agent/migration'
|
||||
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
import { authService } from '@shumai/core/src/auth/auth'
|
||||
@@ -51,6 +52,7 @@ async function run() {
|
||||
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
await migrateLegacyAgentAvatars().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
workflowService.start()
|
||||
if (process.env.WORKFLOW_EXECUTOR === 'temporal') {
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
"group": "Shumai Agent Settings",
|
||||
"pages": [
|
||||
"using-shumai/agent-providers",
|
||||
"using-shumai/image-video-generation",
|
||||
"using-shumai/agents",
|
||||
"using-shumai/agents-md",
|
||||
"using-shumai/agent-skills",
|
||||
|
||||
@@ -5,7 +5,7 @@ description: Configure the tools and capabilities permitted for each agent.
|
||||
|
||||
Shumai agents are equipped with built-in tools that allow them to interact with the file system, view files, extract video frames, and execute commands.
|
||||
|
||||
By default, **all tools are enabled** for new agents. Team Owners can deny specific tools to restrict agent capabilities.
|
||||
By default, most tools are enabled for new agents, while sensitive or destructive tools (such as **Delete Asset**) are disabled by default. Team Owners can toggle specific tools to customize agent capabilities.
|
||||
|
||||
<Warning>
|
||||
**Caution:** Disabling tools can cause agents or skills to fail. For example, disabling the
|
||||
@@ -30,36 +30,63 @@ The following tools can be individually toggled in the agent configuration dialo
|
||||
- **Description:** Allows the agent to load and read skill definitions to perform specialized actions.
|
||||
- **Why it matters:** Enables the agent to inspect the instructions/configurations of custom skills registered in your workspace before executing them.
|
||||
|
||||
### 3. Analyze Image (`analyze_image`)
|
||||
### 3. Read Asset (`read_asset`)
|
||||
|
||||
- **Description:** Allows the agent to view and analyze the visual contents of an image file.
|
||||
- **Why it matters:** Enables multimodal agents to "see" image assets uploaded to Shumai, allowing them to answer comments or write metadata about images.
|
||||
- **Description:** Allows the agent to inspect images, extract still frames from video files, render PDF document pages, and read text/markdown documents.
|
||||
- **Why it matters:** Enables multimodal agents to view visual media assets uploaded to Shumai, extract frame-by-frame video screenshots, inspect document layouts, and answer questions about uploaded creative assets.
|
||||
|
||||
### 4. Video Frame Extraction (`screenshot`)
|
||||
|
||||
- **Description:** Allows the agent to extract still image frames from video files for analysis.
|
||||
- **Why it matters:** Enables the agent to perform frame-by-frame analysis on videos (e.g. for comments, annotation, or search).
|
||||
|
||||
### 5. Browse Shumai (`list_assets`)
|
||||
### 4. Browse Shumai (`list_assets`)
|
||||
|
||||
- **Description:** Allows the agent to list folders and browse files in Shumai.
|
||||
- **Why it matters:** Gives the agent visual mapping of the project structure so it can search for other files or locate related assets.
|
||||
|
||||
### 6. Create Folder (`create_folder`)
|
||||
### 5. Create Folder (`create_folder`)
|
||||
|
||||
- **Description:** Allows the agent to create new folders in Shumai.
|
||||
- **Why it matters:** Needed if the agent organizes outputs or places files in custom subdirectory folders.
|
||||
|
||||
### 7. Create File (`create_file`)
|
||||
### 6. Create File (`create_file`)
|
||||
|
||||
- **Description:** Allows the agent to create and upload new files to Shumai.
|
||||
- **Why it matters:** Allows the agent to generate new assets (e.g. text logs, transcoded clips, reports) and save them directly back into your project.
|
||||
|
||||
### 8. Create Version (`create_version`)
|
||||
### 7. Create Version (`create_version`)
|
||||
|
||||
- **Description:** Allows the agent to upload and stack a new version of an existing file in Shumai.
|
||||
- **Why it matters:** Allows the agent to update files in a clean, non-destructive way by stacking revisions rather than creating duplicate files.
|
||||
|
||||
### 8. Generate Image (`generate_image`)
|
||||
|
||||
- **Description:** Allows the agent to generate still images from text prompts and reference images using configured AI image models.
|
||||
- **Why it matters:** Enables agents to create concept art, mockups, textures, and illustrations directly inside Shumai.
|
||||
- **Prerequisite:** This tool requires at least one image model to be configured and enabled in [Image / Video Generation](/using-shumai/image-video-generation). If no image models are enabled under a configured provider, this tool is omitted from the agent session even if enabled in the agent settings dialog.
|
||||
|
||||
### 9. Generate Video (`generate_video`)
|
||||
|
||||
- **Description:** Allows the agent to generate video clips from text prompts, image animation, or transition keyframes using configured AI video models.
|
||||
- **Why it matters:** Enables agents to produce short animated clips, camera motion shots, and video transitions.
|
||||
- **Prerequisite:** This tool requires at least one video model to be configured and enabled in [Image / Video Generation](/using-shumai/image-video-generation). If no video models are enabled under a configured provider, this tool is omitted from the agent session even if enabled in the agent settings dialog.
|
||||
|
||||
### 10. Download Asset (`download_asset`)
|
||||
|
||||
- **Description:** Allows the agent to download a workspace asset or specific storage S3 key to the local `.pi` directory for processing with local scripts or bash commands.
|
||||
- **Why it matters:** Enables agents executing custom skill scripts to fetch files or intermediate transcode artifacts to local disk.
|
||||
|
||||
### 11. Rename Asset (`rename_asset`)
|
||||
|
||||
- **Description:** Allows the agent to rename an existing file or folder in Shumai.
|
||||
- **Why it matters:** Enables agents to keep project files neatly named and organized according to workspace naming conventions. Enabled by default.
|
||||
|
||||
### 12. Move Assets (`move_assets`)
|
||||
|
||||
- **Description:** Allows the agent to move one or more assets to a destination parent folder.
|
||||
- **Why it matters:** Enables agents to reorganize workspace structures, relocate generated outputs, or tidy up project folders. Enabled by default.
|
||||
|
||||
### 13. Delete Asset (`delete_asset`)
|
||||
|
||||
- **Description:** Allows the agent to delete a single file or folder by moving it to the Trash bin.
|
||||
- **Why it matters:** Provides agents with the ability to clean up outdated or redundant assets. To prevent accidental data loss, this tool is **disabled by default** and strictly limited to deleting one asset at a time. Team Owners must explicitly enable this tool for an agent.
|
||||
|
||||
---
|
||||
|
||||
## Restricting Tools
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
---
|
||||
title: Image / Video Generation
|
||||
description: Configure image and video generation providers and models for built-in AI agent tools.
|
||||
---
|
||||
|
||||
Shumai agents can generate creative visual assets directly within your workspace using built-in agent tools: **Generate Image** (`generate_image`) and **Generate Video** (`generate_video`).
|
||||
|
||||
Because generating visual media requires specialized foundation models, you must configure upstream media providers and enable specific models in your team settings before agents can use these capabilities.
|
||||
|
||||
> 🔒 **Access Control:** Only **Team Owners** can view and configure media generation settings. To access this page, click your avatar, navigate to **Settings**, and select **Image / Video Generation** from the left sidebar (or navigate to `teams/<teamId>/settings#image-video`).
|
||||
|
||||
---
|
||||
|
||||
## Why This Setting Is Needed
|
||||
|
||||
Conversational AI models (configured under **AI Providers**) handle text reasoning and chat interactions, but image and video generation require dedicated multimodal generative models (such as DALL-E, FLUX, Gemini Image, Veo, Kling AI, Luma Ray, Wan, and others).
|
||||
|
||||
The **Image / Video Generation** settings page allows team owners to:
|
||||
|
||||
1. Authenticate with one or more supported image and video generation providers.
|
||||
2. Select which curated models to enable for your team, or register custom model IDs.
|
||||
3. Establish default models for agents when generating visual assets.
|
||||
|
||||
---
|
||||
|
||||
## Tool Availability & Activation Rules
|
||||
|
||||
<Warning>
|
||||
**Important:** Enabling `generate_image` or `generate_video` in an agent's configuration dialog is
|
||||
**not enough** on its own. The tool will only be available to the agent if all three of the
|
||||
following conditions are met:
|
||||
</Warning>
|
||||
|
||||
1. **Provider Configured:** The provider must have a valid API key (either set via server environment variables or configured with a custom team key).
|
||||
2. **At Least One Model Enabled:** There must be **at least one enabled model** for that modality (`image` or `video`) under an active provider.
|
||||
3. **Tool Permitted for Agent:** The tool must not be disabled in the agent's tool permissions list under **Team Settings** > **Agents** > **Tools**.
|
||||
|
||||
### What Happens If No Models Are Configured?
|
||||
|
||||
If no image models are enabled under a configured provider, the `generate_image` tool is **completely omitted** from the agent's runtime environment. Similarly, if no video models are enabled, the `generate_video` tool is omitted.
|
||||
|
||||
In this state:
|
||||
|
||||
- The agent **cannot call** the generation tool.
|
||||
- The tool will not appear in the agent's tool catalog or prompt definition.
|
||||
- Even if a Team Owner has toggled **Generate Image** or **Generate Video** ON in the agent's settings dialog, the agent will not have access to the tool until a model is enabled.
|
||||
|
||||
---
|
||||
|
||||
## Supported Providers
|
||||
|
||||
Shumai provides built-in support for 18 media generation providers:
|
||||
|
||||
| Provider | Supported Modalities | Default Environment Variable |
|
||||
| :-------------------- | :------------------- | :--------------------------- |
|
||||
| **OpenAI** | Image | `OPENAI_API_KEY` |
|
||||
| **Google** | Image, Video | `GEMINI_API_KEY` |
|
||||
| **Google Vertex AI** | Image, Video | `GOOGLE_CLOUD_API_KEY` |
|
||||
| **xAI** | Image, Video | `XAI_API_KEY` |
|
||||
| **Fal** | Image, Video | `FAL_KEY` |
|
||||
| **Replicate** | Image, Video | `REPLICATE_API_TOKEN` |
|
||||
| **Black Forest Labs** | Image, Video | `BFL_API_KEY` |
|
||||
| **Kling AI** | Video | `KLINGAI_API_KEY` |
|
||||
| **Together AI** | Image | `TOGETHER_API_KEY` |
|
||||
| **Fireworks AI** | Image | `FIREWORKS_API_KEY` |
|
||||
| **DeepInfra** | Image | `DEEPINFRA_API_KEY` |
|
||||
| **Luma** | Image | `LUMA_API_KEY` |
|
||||
| **Amazon Bedrock** | Image | `AWS_ACCESS_KEY_ID` |
|
||||
| **Alibaba** | Video | `ALIBABA_API_KEY` |
|
||||
| **ByteDance** | Image, Video | `ARK_API_KEY` |
|
||||
| **MiniMax** | Video | `MINIMAX_API_KEY` |
|
||||
| **Prodia** | Image, Video | `PRODIA_TOKEN` |
|
||||
| **Azure OpenAI** | Image | `AZURE_API_KEY` |
|
||||
|
||||
---
|
||||
|
||||
## Configuring Providers & API Keys
|
||||
|
||||
Clicking on any provider in the **Media Providers** list opens its configuration dialog:
|
||||
|
||||
### Provider Statuses
|
||||
|
||||
- **Configured (Custom)**: A team-specific API key or custom environment variable name was entered in this dialog.
|
||||
- **Configured (Environment)**: The system detected a server-level default environment variable (e.g. `OPENAI_API_KEY`). No team key is required unless you want to override it.
|
||||
- **Not Configured**: No API key is available. You must provide an API key or an environment variable name to use this provider.
|
||||
|
||||
### Overriding API Keys
|
||||
|
||||
In the **API Key** tab of the provider dialog:
|
||||
|
||||
- Enter an API key literal (e.g. `sk-...`) or a custom environment variable name available on the host system.
|
||||
- Click **Clear Custom Key** to revert back to using the default environment variable if one exists.
|
||||
|
||||
---
|
||||
|
||||
## Managing Models
|
||||
|
||||
In the **Models** tab of the provider dialog:
|
||||
|
||||
### Curated Models
|
||||
|
||||
Shumai comes pre-configured with popular curated models (such as `dall-e-3`, `fal-ai/flux/dev`, `veo-3.1-generate`, `kling-v3.0-t2v`, etc.). Toggle the switch next to any model to enable or disable it for your team.
|
||||
|
||||
### Custom Models
|
||||
|
||||
If your provider supports a new or fine-tuned model not listed in the curated models:
|
||||
|
||||
1. Select the modality (**Image** or **Video**).
|
||||
2. Enter the exact **Model ID** recognized by the upstream provider.
|
||||
3. Click **Add**.
|
||||
|
||||
### Default Model Selection
|
||||
|
||||
The **first enabled model** of each type (`image` or `video`) automatically becomes the default model. When an agent calls `generate_image` or `generate_video` without explicitly specifying a `model` parameter, Shumai automatically directs the generation request to this default model.
|
||||
|
||||
---
|
||||
|
||||
## How Agents Use Generation Tools
|
||||
|
||||
Once enabled, agents can invoke the tools during conversations or automated workflows:
|
||||
|
||||
### 1. Generate Image (`generate_image`)
|
||||
|
||||
- **Prompt:** Detailed text description of the image.
|
||||
- **Model Selection:** Automatically defaults to your first enabled image model, or the agent can choose any enabled model in `provider:modelId` format (e.g. `openai:dall-e-3` or `fal:fal-ai/flux/dev`).
|
||||
- **Image References:** Agents can use existing workspace assets as style or composition references by passing S3 storage keys obtained from the `read_asset` tool (`s3KeyOnly: true`).
|
||||
- **Inpainting:** Supports mask-guided localized image editing using an S3 mask key.
|
||||
- **Parameters:** Custom aspect ratios (`1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `2:3`, `3:2`), dimensions, and seeds.
|
||||
|
||||
### 2. Generate Video (`generate_video`)
|
||||
|
||||
- **Generation Modes:**
|
||||
- `text_to_video`: Generate a clip from a descriptive motion prompt.
|
||||
- `image_to_video`: Animate an existing still image asset.
|
||||
- `first_last_frame`: Seamlessly interpolate and transition between two keyframe assets.
|
||||
- `reference_to_video`: Use reference images for consistent characters or environments.
|
||||
- **Parameters:** Aspect ratio (`16:9`, `9:16`, `1:1`, `adaptive`), resolution, duration, frame rate (FPS), and audio generation.
|
||||
|
||||
### Asset Creation Workflow
|
||||
|
||||
All generated media files are initially saved to the local sandboxed directory (`.pi/`). Agents can then use **Create File** (`create_file`) to upload the output as a new project asset or **Create Version** (`create_version`) to stack it onto an existing revision history.
|
||||
@@ -53,7 +53,7 @@ Shumai supports quota enforcement across multiple layers of the AI and agent arc
|
||||
| **AI Cost ($)** | Dollar expenditure accrued from AI model providers. | Team, Role, or Members |
|
||||
| **MCP Server Calls** | Number of tool invocations directed to a connected Model Context Protocol (MCP) server. | Target MCP Server ID |
|
||||
| **Sandboxed Bash Commands** | Shell commands executed by the agent inside the isolated sandbox. | Wildcard pattern (e.g. `npm *`, `*`) |
|
||||
| **Agent Tool Calls** | Calls to specific native tools (e.g. `analyze_image`, `screenshot`, `create_file`). | Tool name |
|
||||
| **Agent Tool Calls** | Calls to specific native tools (e.g. `read_asset`, `download_asset`, `create_file`). | Tool name |
|
||||
|
||||
### Bash Command Wildcard Matching
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ export default tseslint.config(
|
||||
'**/packages/db/src/generated/**',
|
||||
'**/packages/core/src/generated/**',
|
||||
'**/data/**',
|
||||
'**/packages/core/src/provider/generator/**',
|
||||
'**/packages/webui/components/ui/**',
|
||||
'**/packages/webui/paraglide/**',
|
||||
'**/packages/webui/public/**',
|
||||
|
||||
+17
-17
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
@@ -13,8 +13,8 @@
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "bun run --hot apps/web/src/index.ts",
|
||||
"i18n:compile": "paraglide-js compile --project ./packages/webui/project.inlang --outdir ./packages/webui/paraglide --emit-ts-declarations --no-emit-git-ignore",
|
||||
"i18n:watch": "paraglide-js compile --project ./packages/webui/project.inlang --outdir ./packages/webui/paraglide --watch --emit-ts-declarations --no-emit-git-ignore",
|
||||
"i18n:compile": "paraglide-js compile --project ./packages/webui/project.inlang --outdir ./packages/webui/paraglide --emit-ts-declarations",
|
||||
"i18n:watch": "paraglide-js compile --project ./packages/webui/project.inlang --outdir ./packages/webui/paraglide --watch --emit-ts-declarations",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
||||
"test": "bun --bun vitest run",
|
||||
@@ -30,8 +30,8 @@
|
||||
"db:generate": "cd packages/db && bunx prisma generate",
|
||||
"worker:agent": "bun apps/agent/src/index.ts",
|
||||
"worker:transcode": "bun apps/transcode/src/index.ts",
|
||||
"build": "bun run build.ts",
|
||||
"build-npm": "bun scripts/build-npm.ts",
|
||||
"build": "bun run i18n:compile && bun run build.ts",
|
||||
"build-npm": "bun run i18n:compile && bun scripts/build-npm.ts",
|
||||
"prepublishOnly": "bun run build-npm && bun run lint && bun run typecheck",
|
||||
"publish": "bun run prepublishOnly && bun run scripts/publish.ts",
|
||||
"publish:dry": "bun run prepublishOnly && bun run scripts/publish.ts --dry-run",
|
||||
@@ -42,29 +42,29 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "10.0.1",
|
||||
"@inlang/paraglide-js": "2.20.0",
|
||||
"@inlang/paraglide-js": "2.25.0",
|
||||
"@playwright/test": "1.61.1",
|
||||
"@testcontainers/postgresql": "12.0.1",
|
||||
"@testcontainers/postgresql": "12.1.0",
|
||||
"@testing-library/dom": "10.4.1",
|
||||
"@testing-library/react": "16.3.2",
|
||||
"@types/bun": "1.3.14",
|
||||
"@testing-library/react": "16.3.3",
|
||||
"@types/bun": "1.4.0",
|
||||
"@types/node": "25.9.1",
|
||||
"bun-plugin-pino": "1.5.0",
|
||||
"bun-plugin-tailwind": "0.1.2",
|
||||
"bun-types": "1.3.14",
|
||||
"eslint": "10.4.1",
|
||||
"bun-types": "1.4.0",
|
||||
"eslint": "10.9.1",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-plugin-prettier": "5.5.6",
|
||||
"eslint-plugin-react": "7.37.5",
|
||||
"globals": "17.6.0",
|
||||
"happy-dom": "20.10.6",
|
||||
"globals": "17.11.0",
|
||||
"happy-dom": "20.12.0",
|
||||
"pino-pretty": "13.1.3",
|
||||
"prisma": "7.8.0",
|
||||
"testcontainers": "12.0.1",
|
||||
"prisma": "7.10.0",
|
||||
"testcontainers": "12.1.0",
|
||||
"typescript": "6.0.3",
|
||||
"typescript-eslint": "8.60.1",
|
||||
"typescript-eslint": "8.68.0",
|
||||
"vite-tsconfig-paths": "6.1.1",
|
||||
"vitest": "4.1.8"
|
||||
"vitest": "4.1.11"
|
||||
},
|
||||
"patchedDependencies": {
|
||||
"@earendil-works/pi-ai@0.82.1": "patches/@earendil-works%2Fpi-ai@0.82.1.patch"
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.66",
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.74",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@earendil-works/pi-ai": "0.82.1",
|
||||
"@google/genai": "1.52.0",
|
||||
"@google/genai": "2.19.0",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@shumai/workflow-core": "workspace:*",
|
||||
"@temporalio/activity": "1.17.2",
|
||||
"@temporalio/worker": "1.17.2",
|
||||
"@temporalio/workflow": "1.17.2",
|
||||
"adm-zip": "0.5.17",
|
||||
"@temporalio/activity": "1.23.0",
|
||||
"@temporalio/worker": "1.23.0",
|
||||
"@temporalio/workflow": "1.23.0",
|
||||
"adm-zip": "0.6.0",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"typebox": "1.1.38",
|
||||
"typebox": "1.3.22",
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.4.3"
|
||||
"zod": "4.5.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/adm-zip": "0.5.8"
|
||||
|
||||
@@ -711,7 +711,7 @@ describe('Agent Database Activities Integration', () => {
|
||||
expect(threadSession1?.userCommentId).toBe(mainComment1.id) // Thread Session has userCommentId = rootComment.id
|
||||
})
|
||||
|
||||
it('should dynamically tag top-level comments with [Thread ID: id] even if thread was created after initial sync', async () => {
|
||||
it('should dynamically tag top-level comments with thread metadata in context even if thread was created after initial sync', async () => {
|
||||
const msg3 = await prisma.assetComment.create({
|
||||
data: { assetId: asset.id, message: 'Question in msg3', creatorId: user.id },
|
||||
})
|
||||
@@ -756,8 +756,13 @@ describe('Agent Database Activities Integration', () => {
|
||||
const msg3Entry = pathEntries.find((e) => e.id.includes(msg3.id))
|
||||
expect(msg3Entry).toBeDefined()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((msg3Entry as any).details?.thread).toEqual({
|
||||
id: msg3.id,
|
||||
replyCount: 1,
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const text = (msg3Entry as any).content ?? (msg3Entry as any).message?.content[0]?.text
|
||||
expect(text).toContain(`[Thread ID: ${msg3.id}] [Replies: 1]`)
|
||||
expect(text).not.toContain(`[Thread ID: ${msg3.id}]`)
|
||||
})
|
||||
|
||||
it('should create separate thread sessions when agent is mentioned in top-level comments (msg3 & msg5) sharing DAG entries without duplication', async () => {
|
||||
@@ -979,7 +984,7 @@ describe('Agent Database Activities Integration', () => {
|
||||
).resolves.not.toThrow()
|
||||
})
|
||||
|
||||
it('should not tag Thread ID when only __CHAT__ placeholder comment exists under root comment', async () => {
|
||||
it('should not tag thread when only __CHAT__ placeholder comment exists under root comment', async () => {
|
||||
const rootComment = await prisma.assetComment.create({
|
||||
data: { assetId: asset.id, message: 'Root comment asking agent', creatorId: user.id },
|
||||
})
|
||||
@@ -1006,8 +1011,7 @@ describe('Agent Database Activities Integration', () => {
|
||||
|
||||
expect(rootEntry).toBeDefined()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- entry data structure parsing
|
||||
const content = (rootEntry as any)?.message?.content?.[0]?.text || ''
|
||||
expect(content).not.toContain(`[Thread ID: ${rootComment.id}]`)
|
||||
expect((rootEntry as any)?.details?.thread).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should return user name and role for team member', async () => {
|
||||
@@ -1696,5 +1700,77 @@ describe('Agent Database Activities Integration', () => {
|
||||
'<project_instructions path="/subfolder/AGENTS.md">\n# Subfolder Policy\n</project_instructions>',
|
||||
)
|
||||
})
|
||||
|
||||
it('agentChatActivity injects # Comment Threads into systemPrompt when userCommentId is present', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User_Comment_Prompt', email: `comment-prompt-${Date.now()}@example.com` },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team_Comment_Prompt_' + Date.now() },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project_Comment_Prompt', teamId: team.id },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'Asset_1',
|
||||
type: AssetType.file,
|
||||
projectId: project.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
const comment = await prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
message: 'Question',
|
||||
creatorId: user.id,
|
||||
},
|
||||
})
|
||||
|
||||
const mockHarness = {
|
||||
subscribe: vi.fn(),
|
||||
prompt: vi.fn().mockResolvedValue({
|
||||
content: [{ type: 'text', text: 'Chat reply' }],
|
||||
usage: { input: 10, output: 20 },
|
||||
}),
|
||||
}
|
||||
const mockSession = {
|
||||
getEntries: vi.fn().mockResolvedValue([]),
|
||||
getStorage: vi.fn().mockReturnValue({ sessionId: 'mock-session-id' }),
|
||||
}
|
||||
|
||||
vi.mocked(piAgent.createAgentSession).mockResolvedValue({
|
||||
session: mockSession as unknown as Session<DatabaseSessionMetadata>,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock AgentHarness instance for activity test
|
||||
harness: mockHarness as unknown as AgentHarness<any, any, any, any>,
|
||||
})
|
||||
|
||||
const context = {
|
||||
agent: { id: 'agent-1', provider: { name: 'google' }, modelRef: { modelId: 'gemini' } },
|
||||
dbProviders: [],
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
} as unknown as AgentExecutionContext
|
||||
|
||||
await agentChatActivity({
|
||||
teamId: team.id,
|
||||
agentId: 'agent-1',
|
||||
message: 'Hello',
|
||||
imageUrls: [],
|
||||
projectId: project.id,
|
||||
folderId: asset.id,
|
||||
assetId: asset.id,
|
||||
sessionId: 'mock-session-id',
|
||||
userCommentId: comment.id,
|
||||
userId: user.id,
|
||||
context,
|
||||
})
|
||||
|
||||
const lastCall = vi.mocked(piAgent.createAgentSession).mock.calls.at(-1)
|
||||
const systemPromptArg = lastCall?.[0].systemPrompt as string
|
||||
expect(systemPromptArg).toContain('# Comment Threads')
|
||||
expect(systemPromptArg).toContain('<thread id="..." reply_count="..." />')
|
||||
expect(systemPromptArg).toContain('read_thread')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -111,14 +111,14 @@ async function executeAgentPrompt(params: {
|
||||
let systemPrompt = `You are the AI assistant for shumai.
|
||||
shumai is a professional creative collaboration platform similar to frame.io, where users can upload creative files, manage projects, assign tasks, get precise feedback, and share their work. As the agent, your role is to help users use and manage this platform.
|
||||
|
||||
shumai has its own cloud file system. If a user asks you to perform file system operations (for example: creating a folder, creating a file, stacking a version, or listing assets), you MUST use the corresponding agent system tools (e.g., 'create_folder', 'create_file', 'create_version', 'list_assets'). Before calling 'create_folder', first check if the folder exists using 'list_assets'. Do NOT use local bash commands or the local bash tool to perform these operations locally on the host environment; all operations must be executed through the platform's cloud file system tools so they are correctly registered and visible within the platform.
|
||||
shumai has its own cloud file system. If a user asks you to perform file system operations (for example: creating a folder, creating a file, stacking a version, listing assets, renaming an asset, moving assets, or deleting an asset), you MUST use the corresponding agent system tools (e.g., 'create_folder', 'create_file', 'create_version', 'list_assets', 'rename_asset', 'move_assets', 'delete_asset'). Before calling 'create_folder', first check if the folder exists using 'list_assets'. Do NOT use local bash commands or the local bash tool to perform these operations locally on the host environment; all operations must be executed through the platform's cloud file system tools so they are correctly registered and visible within the platform.
|
||||
|
||||
If you need to create files in the local filesystem (for example, a temporary file for uploading), only the '.pi' folder in the current directory has write permissions. Do NOT attempt to create files in any other directories.
|
||||
|
||||
When creating a file or version, you may attach metadata (for example, the AI model or prompt used to generate the asset). The allowed field keys and value types are declared directly in the 'metadata' parameter of the 'create_file' and 'create_version' tools.
|
||||
|
||||
# Message Context & Tools
|
||||
User messages may contain a <context> block detailing the user, active asset location, playback position, and attachments. Use the asset IDs from <context> when invoking tools (e.g. 'analyze_image', 'screenshot', 'read_pdf_pages', 'download_asset', 'list_assets'). When an <annotation /> tag is present, it indicates the user has drawn visual markups on the asset at the specified position.`
|
||||
User messages may contain a <context> block detailing the user, active asset location, playback position, and attachments. Use the asset IDs from <context> when invoking tools (e.g. 'read_asset', 'download_asset', 'list_assets'). When an <annotation id="..." /> tag is present, it indicates the user has drawn visual markups on the asset at the specified position. When calling 'read_asset', pass the 'annotationId' parameter with that ID to view the image or rendered pages with the visual markups overlaid.`
|
||||
|
||||
if (agent.soul) {
|
||||
systemPrompt = `${systemPrompt}\n\nAgent Personality and Core Instructions:\n${agent.soul}`
|
||||
@@ -163,6 +163,21 @@ User messages may contain a <context> block detailing the user, active asset loc
|
||||
systemPrompt += `\n\nYour current model supports the following input types: ${modelConfig.input.join(', ')}.`
|
||||
}
|
||||
|
||||
const isCommentMode =
|
||||
!!params.userCommentId ||
|
||||
(params.sessionId
|
||||
? (
|
||||
await prisma.agentSession.findUnique({
|
||||
where: { id: params.sessionId },
|
||||
select: { type: true },
|
||||
})
|
||||
)?.type === 'comment'
|
||||
: false)
|
||||
|
||||
if (isCommentMode) {
|
||||
systemPrompt += `\n\n# Comment Threads\nWhen operating in comment mode, previous top-level messages in the conversation history may contain a <thread id="..." reply_count="..." /> tag indicating an earlier discussion thread with replies. If a user's question refers to or depends on earlier comments or discussions, you can use the 'read_thread' tool with the thread ID to inspect all replies in that thread.`
|
||||
}
|
||||
|
||||
const agentConfig = agent.config as PrismaJson.AgentConfig | null | undefined
|
||||
const thinkingLevel = agentConfig?.thinkingLevel || 'off'
|
||||
|
||||
@@ -809,9 +824,15 @@ export async function initializeAgentSessionActivity(params: {
|
||||
}
|
||||
|
||||
const commentDetails: ShumaiMessageContext = {
|
||||
id: c.id,
|
||||
...(userObj ? { user: userObj } : {}),
|
||||
...(position ? { position } : {}),
|
||||
...(annotation ? { annotation: true } : {}),
|
||||
...(annotation
|
||||
? {
|
||||
annotation: true,
|
||||
annotations: c.annotation as unknown as ShumaiMessageContext['annotations'],
|
||||
}
|
||||
: {}),
|
||||
...(attachedFiles.length > 0 ? { attachedFiles } : {}),
|
||||
}
|
||||
|
||||
@@ -1190,6 +1211,7 @@ export async function getAssetActivity(assetId: string) {
|
||||
},
|
||||
})
|
||||
|
||||
let resolvedAsset = asset
|
||||
if (asset && asset.type === AssetType.version_stack) {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
@@ -1205,18 +1227,36 @@ export async function getAssetActivity(assetId: string) {
|
||||
})
|
||||
|
||||
if (latestVersion) {
|
||||
return {
|
||||
resolvedAsset = {
|
||||
...latestVersion,
|
||||
project: latestVersion.project ?? asset.project,
|
||||
}
|
||||
} as typeof asset
|
||||
}
|
||||
}
|
||||
|
||||
return asset
|
||||
if (!resolvedAsset) return null
|
||||
|
||||
let url: string | undefined
|
||||
if (resolvedAsset.storageKey?.key) {
|
||||
try {
|
||||
url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
resolvedAsset.storageKey.key,
|
||||
'GET',
|
||||
)
|
||||
} catch {
|
||||
// Ignore presign errors
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...resolvedAsset,
|
||||
url,
|
||||
}
|
||||
}
|
||||
|
||||
export async function getCommentActivity(commentId: string) {
|
||||
return prisma.assetComment.findUnique({
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: commentId },
|
||||
include: {
|
||||
attachments: {
|
||||
@@ -1224,6 +1264,34 @@ export async function getCommentActivity(commentId: string) {
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if (!comment) return null
|
||||
|
||||
const attachmentsWithUrl = await Promise.all(
|
||||
comment.attachments.map(async (att) => {
|
||||
let url: string | undefined
|
||||
if (att.asset?.storageKey?.key) {
|
||||
try {
|
||||
url = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
att.asset.storageKey.key,
|
||||
'GET',
|
||||
)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
return {
|
||||
...att,
|
||||
asset: att.asset ? { ...att.asset, url } : null,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
return {
|
||||
...comment,
|
||||
attachments: attachmentsWithUrl,
|
||||
}
|
||||
}
|
||||
|
||||
export async function getProjectAutofillFieldsActivity(projectId: string) {
|
||||
|
||||
@@ -31,6 +31,25 @@ describe('serializeContextToXml', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('serializes context with user and thread tag with reply_count', () => {
|
||||
const context: ShumaiMessageContext = {
|
||||
user: { id: 'usr_1', name: 'Alice', role: 'editor' },
|
||||
thread: { id: 'th_01JXYZ', replyCount: 3 },
|
||||
}
|
||||
expect(serializeContextToXml(context)).toBe(
|
||||
'<context>\n <user id="usr_1" name="Alice" role="editor" />\n <thread id="th_01JXYZ" reply_count="3" />\n</context>',
|
||||
)
|
||||
})
|
||||
|
||||
it('serializes context with thread tag without reply_count', () => {
|
||||
const context: ShumaiMessageContext = {
|
||||
thread: { id: 'th_01JXYZ' },
|
||||
}
|
||||
expect(serializeContextToXml(context)).toBe(
|
||||
'<context>\n <thread id="th_01JXYZ" />\n</context>',
|
||||
)
|
||||
})
|
||||
|
||||
it('serializes minimal context with only currentAsset (without ancestors)', () => {
|
||||
const context: ShumaiMessageContext = {
|
||||
currentAsset: {
|
||||
@@ -100,7 +119,7 @@ describe('serializeContextToXml', () => {
|
||||
<folder id="ast_01JC03_PARENT" name="Footage" />
|
||||
</ancestors>
|
||||
</current_asset>
|
||||
<position type="time" seconds="14.25" />
|
||||
<position type="time" seconds="14.2500" />
|
||||
<annotation />
|
||||
<attached_files>
|
||||
<file id="ast_01JC05_ATT1" name="brand_guidelines_2026.pdf" type="file" media_type="pdf" path="Brand/brand_guidelines_2026.pdf" />
|
||||
@@ -296,7 +315,7 @@ describe('serializeContextToXml', () => {
|
||||
expect(refOrder).toEqual(['a_asset', 'm_asset', 'z_asset'])
|
||||
})
|
||||
|
||||
it('normalizes float formatting with fixed .toFixed(2)', () => {
|
||||
it('normalizes float formatting with fixed precision (toFixed(4) for position, toFixed(2) for duration)', () => {
|
||||
const context1: ShumaiMessageContext = {
|
||||
currentAsset: { id: '1', name: 'a', type: 'file', durationSeconds: 14 },
|
||||
position: { type: 'time', seconds: 5.5 },
|
||||
@@ -307,9 +326,29 @@ describe('serializeContextToXml', () => {
|
||||
}
|
||||
|
||||
expect(serializeContextToXml(context1)).toContain('duration_seconds="14.00"')
|
||||
expect(serializeContextToXml(context1)).toContain('seconds="5.50"')
|
||||
expect(serializeContextToXml(context1)).toContain('seconds="5.5000"')
|
||||
expect(serializeContextToXml(context2)).toContain('duration_seconds="14.00"')
|
||||
expect(serializeContextToXml(context2)).toContain('seconds="5.50"')
|
||||
expect(serializeContextToXml(context2)).toContain('seconds="5.5000"')
|
||||
})
|
||||
|
||||
it('serializes annotation tag with id when context.id is provided', () => {
|
||||
const context: ShumaiMessageContext = {
|
||||
id: '01JABCDEF1234567890',
|
||||
annotation: true,
|
||||
}
|
||||
expect(serializeContextToXml(context)).toBe(
|
||||
'<context>\n <annotation id="01JABCDEF1234567890" />\n</context>',
|
||||
)
|
||||
})
|
||||
|
||||
it('escapes special XML characters in annotation id', () => {
|
||||
const context: ShumaiMessageContext = {
|
||||
id: 'test<id>&"123"',
|
||||
annotation: true,
|
||||
}
|
||||
expect(serializeContextToXml(context)).toBe(
|
||||
'<context>\n <annotation id="test<id>&"123"" />\n</context>',
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -7,8 +7,7 @@ import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { createAgentSession, type DbProviderInfo } from './index'
|
||||
import * as sandboxedBashModule from './tools/sandboxed-bash'
|
||||
import * as analyzeImageModule from './tools/analyze-image'
|
||||
import * as screenshotModule from './tools/screenshot'
|
||||
import * as readAssetModule from './tools/read-asset'
|
||||
|
||||
describe('DatabaseSessionStorage', () => {
|
||||
setupTestDbHooks()
|
||||
@@ -451,9 +450,8 @@ describe('DatabaseSessionStorage', () => {
|
||||
})
|
||||
const sessionId = initialStorage.sessionId
|
||||
|
||||
// 2. Spy on media tools constructors
|
||||
const createAnalyzeImageToolSpy = vi.spyOn(analyzeImageModule, 'createAnalyzeImageTool')
|
||||
const createScreenshotToolSpy = vi.spyOn(screenshotModule, 'createScreenshotTool')
|
||||
// 2. Spy on media tool constructor
|
||||
const createReadAssetToolSpy = vi.spyOn(readAssetModule, 'createReadAssetTool')
|
||||
|
||||
// 3. Resume the session via createAgentSession with a NEW comment ID
|
||||
await createAgentSession({
|
||||
@@ -470,12 +468,11 @@ describe('DatabaseSessionStorage', () => {
|
||||
providers: mockProviders,
|
||||
})
|
||||
|
||||
// 4. Assert that the tools were instantiated using the new comment ID and user ID
|
||||
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
// 4. Assert that the tool was instantiated using user ID
|
||||
expect(createReadAssetToolSpy).toHaveBeenCalledWith(user.id)
|
||||
})
|
||||
|
||||
it('should use the current comment ID for media tools in subsequent turns (video)', async () => {
|
||||
it('should instantiate media tools with user ID in subsequent turns (video)', async () => {
|
||||
const { agent, user, team } = await setupTestData()
|
||||
|
||||
// Create a project and asset to satisfy FK constraints
|
||||
@@ -505,9 +502,8 @@ describe('DatabaseSessionStorage', () => {
|
||||
})
|
||||
const sessionId = initialStorage.sessionId
|
||||
|
||||
// 2. Spy on media tools constructors
|
||||
const createAnalyzeImageToolSpy = vi.spyOn(analyzeImageModule, 'createAnalyzeImageTool')
|
||||
const createScreenshotToolSpy = vi.spyOn(screenshotModule, 'createScreenshotTool')
|
||||
// 2. Spy on media tool constructor
|
||||
const createReadAssetToolSpy = vi.spyOn(readAssetModule, 'createReadAssetTool')
|
||||
|
||||
// 3. Resume the session via createAgentSession with a NEW comment ID
|
||||
await createAgentSession({
|
||||
@@ -524,9 +520,8 @@ describe('DatabaseSessionStorage', () => {
|
||||
providers: mockProviders,
|
||||
})
|
||||
|
||||
// 4. Assert that the tools were instantiated using the new comment ID and user ID
|
||||
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
|
||||
// 4. Assert that the tool was instantiated using user ID
|
||||
expect(createReadAssetToolSpy).toHaveBeenCalledWith(user.id)
|
||||
})
|
||||
|
||||
describe('P1 Bug Reproductions', () => {
|
||||
@@ -742,5 +737,78 @@ describe('DatabaseSessionStorage', () => {
|
||||
const stats = await storage.getSessionStats()
|
||||
expect(stats.messageCount).toBe(1)
|
||||
})
|
||||
|
||||
it('should dynamically inject thread metadata into details.thread in getPathToRoot when replies exist', async () => {
|
||||
const { agent, user } = await setupTestData()
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Test Project', teamId: agent.teamId },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'Test Asset',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
const storage = await DatabaseSessionStorage.create({ agentId: agent.id, assetId: asset.id })
|
||||
|
||||
const rootComment = await prisma.assetComment.create({
|
||||
data: {
|
||||
id: 'comment-root-1',
|
||||
assetId: asset.id,
|
||||
message: 'Top-level question',
|
||||
creatorId: user.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Add a reply
|
||||
await prisma.assetComment.create({
|
||||
data: {
|
||||
id: 'comment-reply-1',
|
||||
assetId: asset.id,
|
||||
message: 'Reply 1',
|
||||
creatorId: user.id,
|
||||
replyToId: rootComment.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Create session entry for root comment
|
||||
await prisma.agentSessionEntry.create({
|
||||
data: {
|
||||
id: rootComment.id,
|
||||
sessionId: storage.sessionId,
|
||||
assetId: asset.id,
|
||||
type: 'custom_message',
|
||||
data: {
|
||||
customType: 'shumai_message',
|
||||
content: 'Top-level question',
|
||||
display: true,
|
||||
details: { user: { id: user.id, name: user.name } },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const path = await storage.getPathToRoot(rootComment.id)
|
||||
expect(path).toHaveLength(1)
|
||||
expect(path[0].type).toBe('custom_message')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const details = (path[0] as any).details
|
||||
expect(details.thread).toEqual({
|
||||
id: rootComment.id,
|
||||
replyCount: 1,
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((path[0] as any).content).toBe('Top-level question')
|
||||
|
||||
const entries = await storage.getEntries()
|
||||
expect(entries).toHaveLength(1)
|
||||
expect(entries[0].type).toBe('custom_message')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((entries[0] as any).details?.thread).toEqual({
|
||||
id: rootComment.id,
|
||||
replyCount: 1,
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type SessionTreeEntry,
|
||||
} from '@earendil-works/pi-agent-core'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import type { ShumaiMessageContext } from '@shumai/dtos'
|
||||
|
||||
export interface DatabaseSessionMetadata extends SessionMetadata {
|
||||
agentId: string
|
||||
@@ -131,7 +132,10 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
customType: 'shumai_message',
|
||||
content: contentText,
|
||||
display: true,
|
||||
details: this.currentMessageContext,
|
||||
details: {
|
||||
...this.currentMessageContext,
|
||||
id: entryToProcess.id,
|
||||
},
|
||||
} as unknown as SessionTreeEntry
|
||||
this.currentMessageContext = undefined
|
||||
}
|
||||
@@ -366,59 +370,7 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
await this.reinjectSkillContentAsync(entry)
|
||||
}
|
||||
|
||||
// Dynamically tag comments that currently have reply threads
|
||||
const entryIds = pathEntries.map((e) => e.id)
|
||||
if (entryIds.length > 0) {
|
||||
const threadCounts = await prisma.assetComment.groupBy({
|
||||
by: ['replyToId'],
|
||||
where: {
|
||||
replyToId: { in: entryIds },
|
||||
message: { not: '__CHAT__' },
|
||||
},
|
||||
_count: {
|
||||
id: true,
|
||||
},
|
||||
})
|
||||
const threadReplyCountMap = new Map<string, number>()
|
||||
for (const tc of threadCounts) {
|
||||
if (tc.replyToId) {
|
||||
threadReplyCountMap.set(tc.replyToId, tc._count.id)
|
||||
}
|
||||
}
|
||||
|
||||
if (threadReplyCountMap.size > 0) {
|
||||
for (let i = 0; i < pathEntries.length; i++) {
|
||||
const entry = pathEntries[i]
|
||||
const count = threadReplyCountMap.get(entry.id)
|
||||
if (count !== undefined && count > 0) {
|
||||
if (entry.type === 'message' && entry.message) {
|
||||
const cloned = structuredClone(entry)
|
||||
const msg = cloned.message as unknown as {
|
||||
content?: Array<{ type: string; text: string }>
|
||||
}
|
||||
if (Array.isArray(msg.content) && msg.content[0] && msg.content[0].type === 'text') {
|
||||
const threadTag = `[Thread ID: ${entry.id}] [Replies: ${count}]`
|
||||
if (!msg.content[0].text.includes(`[Thread ID: ${entry.id}]`)) {
|
||||
msg.content[0].text = `${threadTag} ${msg.content[0].text}`
|
||||
}
|
||||
}
|
||||
pathEntries[i] = cloned
|
||||
} else if (entry.type === 'custom_message') {
|
||||
const cloned = structuredClone(entry)
|
||||
const threadTag = `[Thread ID: ${entry.id}] [Replies: ${count}]`
|
||||
if (typeof cloned.content === 'string') {
|
||||
if (!cloned.content.includes(`[Thread ID: ${entry.id}]`)) {
|
||||
cloned.content = `${threadTag} ${cloned.content}`
|
||||
}
|
||||
}
|
||||
pathEntries[i] = cloned
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pathEntries
|
||||
return this.injectThreadMetadataAsync(pathEntries)
|
||||
}
|
||||
|
||||
async getEntries(): Promise<SessionTreeEntry[]> {
|
||||
@@ -431,6 +383,56 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
await this.reinjectImageDataAsync(entry)
|
||||
await this.reinjectSkillContentAsync(entry)
|
||||
}
|
||||
return this.injectThreadMetadataAsync(entries)
|
||||
}
|
||||
|
||||
private async injectThreadMetadataAsync(
|
||||
entries: SessionTreeEntry[],
|
||||
): Promise<SessionTreeEntry[]> {
|
||||
const entryIds = entries.map((e) => e.id)
|
||||
if (entryIds.length === 0) {
|
||||
return entries
|
||||
}
|
||||
|
||||
const threadCounts = await prisma.assetComment.groupBy({
|
||||
by: ['replyToId'],
|
||||
where: {
|
||||
replyToId: { in: entryIds },
|
||||
message: { not: '__CHAT__' },
|
||||
},
|
||||
_count: {
|
||||
id: true,
|
||||
},
|
||||
})
|
||||
const threadReplyCountMap = new Map<string, number>()
|
||||
for (const tc of threadCounts) {
|
||||
if (tc.replyToId) {
|
||||
threadReplyCountMap.set(tc.replyToId, tc._count.id)
|
||||
}
|
||||
}
|
||||
|
||||
if (threadReplyCountMap.size > 0) {
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const entry = entries[i]
|
||||
const count = threadReplyCountMap.get(entry.id)
|
||||
if (count !== undefined && count > 0) {
|
||||
if (entry.type === 'custom_message') {
|
||||
const cloned = structuredClone(entry)
|
||||
const existingDetails = ((cloned as unknown as { details?: ShumaiMessageContext })
|
||||
.details || {}) as ShumaiMessageContext
|
||||
;(cloned as unknown as { details: ShumaiMessageContext }).details = {
|
||||
...existingDetails,
|
||||
thread: {
|
||||
id: entry.id,
|
||||
replyCount: count,
|
||||
},
|
||||
}
|
||||
entries[i] = cloned
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { createSandboxedBashTool } from './tools/sandboxed-bash'
|
||||
import { SandboxManager, type SandboxAskCallback } from '@anthropic-ai/sandbox-runtime'
|
||||
import { quotaService } from '@shumai/core/src/quota/quota-service'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { type TSchema } from 'typebox'
|
||||
@@ -1028,6 +1029,64 @@ describe('createAgentSession typed metadata schema', () => {
|
||||
expect('metadata' in fileProps).toBe(false)
|
||||
expect('metadata' in versionProps).toBe(false)
|
||||
})
|
||||
|
||||
it('should include rename_asset and move_assets by default, but exclude delete_asset', async () => {
|
||||
const { harness } = await setupSessionWithProject({ withCreationFields: false })
|
||||
const toolNames = harness.getTools().map((t) => t.name)
|
||||
|
||||
expect(toolNames).toContain('rename_asset')
|
||||
expect(toolNames).toContain('move_assets')
|
||||
expect(toolNames).not.toContain('delete_asset')
|
||||
})
|
||||
|
||||
it('should include delete_asset when deniedTools is explicitly empty', async () => {
|
||||
const team = await prisma.team.create({ data: { name: 'Delete Asset Team' } })
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Owner', email: 'owner-del@shumai.ai' },
|
||||
})
|
||||
await prisma.teamMember.create({
|
||||
data: { teamId: team.id, userId: user.id, role: 'owner' },
|
||||
})
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'del-agent',
|
||||
name: 'Del Agent',
|
||||
email: 'del-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'del-agent',
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
deniedTools: [],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
try {
|
||||
const { harness } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
agentId: 'del-agent',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: mockProviders,
|
||||
userId: user.id,
|
||||
})
|
||||
const toolNames = harness.getTools().map((t) => t.name)
|
||||
expect(toolNames).toContain('delete_asset')
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('createAgentSession enabled skill filtering', () => {
|
||||
@@ -1257,3 +1316,240 @@ describe('createAgentSession enabled skill filtering', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('getModelFromDb', () => {
|
||||
it('should prioritize provider active api protocol over model config api', () => {
|
||||
const providers: DbProviderInfo[] = [
|
||||
{
|
||||
name: 'custom-provider',
|
||||
config: { api: 'anthropic-messages', apiKey: 'KEY' },
|
||||
models: [
|
||||
{
|
||||
modelId: 'my-model',
|
||||
name: 'My Model',
|
||||
config: {
|
||||
api: 'openai-responses', // stale model config
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
const model = getModelFromDb(providers, 'custom-provider', 'my-model')
|
||||
expect(model.api).toBe('anthropic-messages')
|
||||
})
|
||||
})
|
||||
|
||||
describe('createAgentSession media generation tools injection', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
it('omits generate_image and generate_video when no valid models are configured', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'No Media Team' },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User 1', email: 'user1@example.com', password: 'pw' },
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
id: 'agent-user-no-media',
|
||||
name: 'Agent User 1',
|
||||
email: 'agent1@test.com',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'google', model: 'gemini' },
|
||||
},
|
||||
})
|
||||
|
||||
vi.spyOn(mediaGenerationService, 'getValidModels').mockResolvedValue({
|
||||
imageModels: [],
|
||||
videoModels: [],
|
||||
providerKeys: {},
|
||||
})
|
||||
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
try {
|
||||
const { harness } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
agentId: agent.id,
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
enabledSkillIds: [],
|
||||
allowedDomains: [],
|
||||
providers: mockProviders,
|
||||
})
|
||||
|
||||
const tools = harness.getTools()
|
||||
expect(tools.find((t) => t.name === 'generate_image')).toBeUndefined()
|
||||
expect(tools.find((t) => t.name === 'generate_video')).toBeUndefined()
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('injects generate_image and generate_video when valid models are present', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Media Team' },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User 2', email: 'user2@example.com', password: 'pw' },
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
id: 'agent-user-with-media',
|
||||
name: 'Agent User 2',
|
||||
email: 'agent2@test.com',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'google', model: 'gemini' },
|
||||
},
|
||||
})
|
||||
|
||||
vi.spyOn(mediaGenerationService, 'getValidModels').mockResolvedValue({
|
||||
imageModels: [
|
||||
{
|
||||
id: 'im-1',
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
videoModels: [
|
||||
{
|
||||
id: 'vm-1',
|
||||
type: 'video',
|
||||
provider: 'klingai',
|
||||
modelId: 'kling-v1-standard',
|
||||
name: 'Kling v1 Standard',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
providerKeys: {
|
||||
openai: { apiKey: 'key-1' },
|
||||
klingai: { apiKey: 'key-2' },
|
||||
},
|
||||
})
|
||||
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
try {
|
||||
const { harness } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
agentId: agent.id,
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
enabledSkillIds: [],
|
||||
allowedDomains: [],
|
||||
providers: mockProviders,
|
||||
})
|
||||
|
||||
const tools = harness.getTools()
|
||||
expect(tools.find((t) => t.name === 'generate_image')).toBeDefined()
|
||||
expect(tools.find((t) => t.name === 'generate_video')).toBeDefined()
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('respects deniedTools when generate_image is denied in agent config', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Denied Media Team' },
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User 3', email: 'user3@example.com', password: 'pw' },
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
id: 'agent-user-denied-media',
|
||||
name: 'Agent User 3',
|
||||
email: 'agent3@test.com',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'google',
|
||||
model: 'gemini',
|
||||
deniedTools: ['generate_image'],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
vi.spyOn(mediaGenerationService, 'getValidModels').mockResolvedValue({
|
||||
imageModels: [
|
||||
{
|
||||
id: 'im-1',
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
videoModels: [
|
||||
{
|
||||
id: 'vm-1',
|
||||
type: 'video',
|
||||
provider: 'klingai',
|
||||
modelId: 'kling-v1-standard',
|
||||
name: 'Kling v1 Standard',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
providerKeys: {
|
||||
openai: { apiKey: 'key-1' },
|
||||
klingai: { apiKey: 'key-2' },
|
||||
},
|
||||
})
|
||||
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
try {
|
||||
const { harness } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
agentId: agent.id,
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
enabledSkillIds: [],
|
||||
allowedDomains: [],
|
||||
providers: mockProviders,
|
||||
})
|
||||
|
||||
const tools = harness.getTools()
|
||||
expect(tools.find((t) => t.name === 'generate_image')).toBeUndefined()
|
||||
expect(tools.find((t) => t.name === 'generate_video')).toBeDefined()
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
+38
-18
@@ -19,19 +19,23 @@ import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { createAnalyzeImageTool } from './tools/analyze-image'
|
||||
import { createCreateFileTool } from './tools/create-file'
|
||||
import { createCreateFolderTool } from './tools/create-folder'
|
||||
import { createCreateVersionTool } from './tools/create-version'
|
||||
import { createDownloadAssetTool } from './tools/download-asset'
|
||||
import { createListAssetsTool } from './tools/list-assets'
|
||||
import { createReadPdfPagesTool } from './tools/read-pdf-pages'
|
||||
import { createReadAssetTool } from './tools/read-asset'
|
||||
import { createReadSkillTool } from './tools/read-skill'
|
||||
import { createReadThreadTool } from './tools/read-thread'
|
||||
import { createSandboxedBashTool } from './tools/sandboxed-bash'
|
||||
import { createScreenshotTool } from './tools/screenshot'
|
||||
import { createGenerateImageTool } from './tools/generate-image'
|
||||
import { createGenerateVideoTool } from './tools/generate-video'
|
||||
import { createRenameAssetTool } from './tools/rename-asset'
|
||||
import { createMoveAssetsTool } from './tools/move-assets'
|
||||
import { createDeleteAssetTool } from './tools/delete-asset'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { serializeContextToXml } from './context/serialize-context'
|
||||
import type { ShumaiMessageContext } from '@shumai/dtos'
|
||||
import { DEFAULT_DENIED_TOOLS, type ShumaiMessageContext } from '@shumai/dtos'
|
||||
|
||||
export interface DbModelInfo {
|
||||
modelId: string
|
||||
@@ -72,10 +76,10 @@ export function getModelFromDb(providers: DbProviderInfo[], providerName: string
|
||||
if (dbProvider.config.baseUrl) {
|
||||
m.baseUrl = dbProvider.config.baseUrl
|
||||
}
|
||||
Object.assign(m, dbModel.config)
|
||||
if (dbProvider.config.api) {
|
||||
m.api = dbProvider.config.api
|
||||
}
|
||||
Object.assign(m, dbModel.config)
|
||||
if (dbModel.name) {
|
||||
m.name = dbModel.name
|
||||
}
|
||||
@@ -312,24 +316,31 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = await storage.getMetadata()
|
||||
const userCommentId =
|
||||
passedUserCommentId !== undefined ? passedUserCommentId : metadata.userCommentId
|
||||
|
||||
const mediaTools: AgentTool[] = []
|
||||
if (userId) {
|
||||
mediaTools.push(
|
||||
createAnalyzeImageTool(userId, userCommentId),
|
||||
createScreenshotTool(userId, userCommentId),
|
||||
createReadPdfPagesTool(userId, userCommentId),
|
||||
)
|
||||
mediaTools.push(createReadAssetTool(userId))
|
||||
}
|
||||
|
||||
const agent = await prisma.agent.findUnique({
|
||||
where: { id: agentId },
|
||||
})
|
||||
const agentConfig = agent?.config as PrismaJson.AgentConfig | null | undefined
|
||||
const deniedTools = agentConfig?.deniedTools || []
|
||||
const deniedTools = agentConfig?.deniedTools ?? [...DEFAULT_DENIED_TOOLS]
|
||||
|
||||
if (!deniedTools.includes('generate_image') || !deniedTools.includes('generate_video')) {
|
||||
try {
|
||||
const { imageModels, videoModels, providerKeys } =
|
||||
await mediaGenerationService.getValidModels(teamId)
|
||||
if (imageModels.length > 0 && !deniedTools.includes('generate_image')) {
|
||||
mediaTools.push(createGenerateImageTool(imageModels, providerKeys, userId))
|
||||
}
|
||||
if (videoModels.length > 0 && !deniedTools.includes('generate_video')) {
|
||||
mediaTools.push(createGenerateVideoTool(videoModels, providerKeys, userId))
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error({ err, teamId }, 'Failed to initialize media generation tools')
|
||||
}
|
||||
}
|
||||
|
||||
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs, {
|
||||
teamId,
|
||||
@@ -372,12 +383,16 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
const systemTools: AgentTool[] = []
|
||||
if (userId) {
|
||||
const metadataSchema = await buildProjectMetadataSchema(projectId)
|
||||
const agentContext = { teamId, agentId }
|
||||
systemTools.push(
|
||||
createListAssetsTool(userId),
|
||||
createCreateFolderTool(userId),
|
||||
createCreateFileTool(userId, metadataSchema),
|
||||
createCreateVersionTool(userId, metadataSchema),
|
||||
createCreateFolderTool(userId, agentContext),
|
||||
createCreateFileTool(userId, metadataSchema, agentContext),
|
||||
createCreateVersionTool(userId, metadataSchema, agentContext),
|
||||
createDownloadAssetTool(userId),
|
||||
createRenameAssetTool(userId, agentContext),
|
||||
createMoveAssetsTool(userId, agentContext),
|
||||
createDeleteAssetTool(userId, agentContext),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -669,3 +684,8 @@ export * from './workflows/agent-autofill'
|
||||
export * from './workflows/agent-chat'
|
||||
export * from './workflows/agent-embedding'
|
||||
export * from './workflows/query-embedding-for-search'
|
||||
export * from './tools/generate-image'
|
||||
export * from './tools/generate-video'
|
||||
export * from './tools/rename-asset'
|
||||
export * from './tools/move-assets'
|
||||
export * from './tools/delete-asset'
|
||||
|
||||
@@ -76,7 +76,13 @@ describe('reproduce null coercion in harness tool validation', () => {
|
||||
|
||||
// execute must accept the valid call (data-only) without throwing
|
||||
const result = await tool.execute('call-1', validated as never)
|
||||
expect(result.details).toEqual({ id: 'file-1', name: 'test', type: 'file', size: 100 })
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
id: 'file-1',
|
||||
name: 'test',
|
||||
type: 'file',
|
||||
size: 100,
|
||||
})
|
||||
expect(s3Service.putObject).toHaveBeenCalledTimes(1)
|
||||
expect(authzService.hasPermission).toHaveBeenCalledTimes(1)
|
||||
expect(assetService.createFile).toHaveBeenCalledTimes(1)
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createAnalyzeImageTool } from './analyze-image'
|
||||
import {
|
||||
prisma,
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type AssetComment,
|
||||
type User,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
assetComment: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
workflowTask: {
|
||||
create: vi.fn(),
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
getObject: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/workflow-core', () => ({
|
||||
workflowService: {
|
||||
executeWait: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('analyzeImageTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should check user permissions and fetch the image from S3 when comment has no annotations', async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
storageKey: { key: 'raw-image.png' },
|
||||
media: null,
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
annotation: null,
|
||||
} as unknown as AssetComment)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-image-bytes'),
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createAnalyzeImageTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1' })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(result.content[0].type).toBe('image')
|
||||
expect((result.content[0] as unknown as { data: string }).data).toBe(
|
||||
Buffer.from('fake-image-bytes').toString('base64'),
|
||||
)
|
||||
expect(result.details.sourceKeys).toEqual(['raw-image.png'])
|
||||
|
||||
expect(prisma.workflowTask.create).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should trigger overlay transcode workflow when comment has annotations', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
storageKey: { key: 'raw-image.png' },
|
||||
media: {
|
||||
imageTranscodes: [{ key: 'transcoded-image.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
annotation: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
} as unknown as AssetComment)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: { key: 'annotated-image.webp' },
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-annotated-image-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createAnalyzeImageTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1' })
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
type: 'transcode_image_annotation',
|
||||
status: 'pending',
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
imageAnnotation: {
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.content[0].type).toBe('image')
|
||||
expect((result.content[0] as unknown as { data: string }).data).toBe(
|
||||
Buffer.from('fake-annotated-image-bytes').toString('base64'),
|
||||
)
|
||||
expect(result.details.sourceKeys).toEqual(['annotated-image.webp'])
|
||||
})
|
||||
})
|
||||
@@ -1,139 +0,0 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
const analyzeImageSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'The asset ID of the image to analyze. This parameter is required.',
|
||||
}),
|
||||
})
|
||||
|
||||
export function createAnalyzeImageTool(
|
||||
userId: string,
|
||||
userCommentId?: string | null,
|
||||
): AgentTool<typeof analyzeImageSchema> {
|
||||
return {
|
||||
name: 'analyze_image',
|
||||
label: 'Analyze Image',
|
||||
description:
|
||||
'Retrieves and views the image content. Call this tool if you need to analyze the image.',
|
||||
parameters: analyzeImageSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const assetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
let asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
if (asset.type === 'version_stack') {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (latestVersion) {
|
||||
asset = latestVersion
|
||||
}
|
||||
}
|
||||
|
||||
// Get default media key
|
||||
let mediaKey = asset.storageKey?.key
|
||||
if (asset.media) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const mediaInfo = asset.media as any
|
||||
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
|
||||
}
|
||||
}
|
||||
|
||||
if (!mediaKey) {
|
||||
throw new Error('No media content found for this asset.')
|
||||
}
|
||||
|
||||
// Check if trigger comment has draw annotations for this asset
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === assetId && comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let keyToUse = mediaKey
|
||||
|
||||
if (annotations) {
|
||||
// Trigger transcode workflow to draw annotation
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
imageAnnotation: {
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
if (output?.key) {
|
||||
keyToUse = output.key
|
||||
} else {
|
||||
throw new Error('Image annotation transcode workflow failed to output S3 key.')
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve the image content from S3
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
|
||||
|
||||
const mimeType =
|
||||
contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp'
|
||||
|
||||
const content: ImageContent[] = [
|
||||
{
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType,
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys: [keyToUse],
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { resolveAnnotationsById } from './annotation-resolver'
|
||||
import { DatabaseSessionStorage } from '../database-session-storage'
|
||||
import type { SessionTreeEntry } from '@earendil-works/pi-agent-core'
|
||||
|
||||
describe('resolveAnnotationsById', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
beforeEach(() => {
|
||||
// DB hooks automatically manage transactions
|
||||
})
|
||||
|
||||
it('returns null annotations and null timestamp when annotationId is undefined or empty', async () => {
|
||||
const result1 = await resolveAnnotationsById('asset-1', undefined)
|
||||
expect(result1).toEqual({ annotations: null, timestamp: null })
|
||||
|
||||
const result2 = await resolveAnnotationsById('asset-1', '')
|
||||
expect(result2).toEqual({ annotations: null, timestamp: null })
|
||||
})
|
||||
|
||||
it('resolves annotations and timestamp from AssetComment (Comments Mode)', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Comment Author', email: 'author@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Test Project', teamId: team.id },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: 'file',
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const comment = await prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
creatorId: user.id,
|
||||
message: 'Look at the highlighted area',
|
||||
second: 18.5,
|
||||
annotation: [
|
||||
{
|
||||
type: 'line',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[10, 10],
|
||||
[50, 50],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
const result = await resolveAnnotationsById(asset.id, comment.id)
|
||||
expect(result.timestamp).toBe(18.5)
|
||||
expect(result.annotations).toEqual([
|
||||
{
|
||||
type: 'line',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[10, 10],
|
||||
[50, 50],
|
||||
],
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('throws an error when AssetComment belongs to a different asset', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Author', email: 'author2@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project', teamId: team.id },
|
||||
})
|
||||
const assetA = await prisma.asset.create({
|
||||
data: { name: 'videoA.mp4', type: 'file', projectId: project.id, status: 'uploaded' },
|
||||
})
|
||||
const assetB = await prisma.asset.create({
|
||||
data: { name: 'videoB.mp4', type: 'file', projectId: project.id, status: 'uploaded' },
|
||||
})
|
||||
|
||||
const commentB = await prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: assetB.id,
|
||||
creatorId: user.id,
|
||||
message: 'Comment on B',
|
||||
annotation: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[10, 10],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
await expect(resolveAnnotationsById(assetA.id, commentB.id)).rejects.toThrow(
|
||||
`Annotation "${commentB.id}" belongs to asset "${assetB.id}", not target asset "${assetA.id}".`,
|
||||
)
|
||||
})
|
||||
|
||||
it('resolves annotations and timestamp from AgentSessionEntry (1-on-1 Chat Mode)', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Chat User', email: 'chatuser@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Chat Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Chat Project', teamId: team.id },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'chat-video.mp4',
|
||||
type: 'file',
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: { name: 'Chat Bot User', email: 'bot@example.com', type: 'agent' },
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'test-provider', model: 'test-model' },
|
||||
},
|
||||
})
|
||||
|
||||
// Create session storage
|
||||
const storage = await DatabaseSessionStorage.create({
|
||||
agentId: agent.id,
|
||||
userId: user.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
|
||||
// Set active message context with markup annotations and position
|
||||
storage.currentMessageContext = {
|
||||
currentAsset: {
|
||||
id: asset.id,
|
||||
name: asset.name,
|
||||
type: 'file',
|
||||
path: asset.name,
|
||||
},
|
||||
position: { type: 'time', seconds: 12.34 },
|
||||
annotation: true,
|
||||
annotations: [
|
||||
{
|
||||
type: 'rectangle',
|
||||
x: 100,
|
||||
y: 200,
|
||||
width: 50,
|
||||
height: 50,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
// User sends a message -> DatabaseSessionStorage.appendEntry is called
|
||||
const userEntry: SessionTreeEntry = {
|
||||
type: 'message',
|
||||
id: '01JTESTENTRY1234567890',
|
||||
parentId: null,
|
||||
timestamp: new Date().toISOString(),
|
||||
message: {
|
||||
role: 'user',
|
||||
content: 'Check this rectangle markup on video',
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
}
|
||||
|
||||
await storage.appendEntry(userEntry)
|
||||
|
||||
// Verify DatabaseSessionStorage persisted entry with details.id = entry.id
|
||||
const dbEntry = await prisma.agentSessionEntry.findUnique({
|
||||
where: { id: '01JTESTENTRY1234567890' },
|
||||
})
|
||||
expect(dbEntry).toBeDefined()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const details = (dbEntry?.data as any)?.details
|
||||
expect(details.id).toBe('01JTESTENTRY1234567890')
|
||||
expect(details.position).toEqual({ type: 'time', seconds: 12.34 })
|
||||
expect(details.annotations).toHaveLength(1)
|
||||
|
||||
// Now resolveAnnotationsById directly with the entry id and matching asset.id
|
||||
const result = await resolveAnnotationsById(asset.id, '01JTESTENTRY1234567890')
|
||||
expect(result.timestamp).toBe(12.34)
|
||||
expect(result.annotations).toEqual([
|
||||
{
|
||||
type: 'rectangle',
|
||||
x: 100,
|
||||
y: 200,
|
||||
width: 50,
|
||||
height: 50,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('throws an error when AgentSessionEntry belongs to a different asset', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User', email: 'user3@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project', teamId: team.id },
|
||||
})
|
||||
const assetA = await prisma.asset.create({
|
||||
data: { name: 'chatA.mp4', type: 'file', projectId: project.id, status: 'uploaded' },
|
||||
})
|
||||
const assetB = await prisma.asset.create({
|
||||
data: { name: 'chatB.mp4', type: 'file', projectId: project.id, status: 'uploaded' },
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: { name: 'Bot User', email: 'bot2@example.com', type: 'agent' },
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'test-provider', model: 'test-model' },
|
||||
},
|
||||
})
|
||||
|
||||
const storage = await DatabaseSessionStorage.create({
|
||||
agentId: agent.id,
|
||||
userId: user.id,
|
||||
assetId: assetB.id,
|
||||
})
|
||||
storage.currentMessageContext = {
|
||||
currentAsset: { id: assetB.id, name: assetB.name, type: 'file', path: assetB.name },
|
||||
annotation: true,
|
||||
annotations: [{ type: 'box' }],
|
||||
}
|
||||
const userEntry: SessionTreeEntry = {
|
||||
type: 'message',
|
||||
id: '01JENTRYONASSETB123456',
|
||||
parentId: null,
|
||||
timestamp: new Date().toISOString(),
|
||||
message: { role: 'user', content: 'Markup on B', timestamp: Date.now() },
|
||||
}
|
||||
await storage.appendEntry(userEntry)
|
||||
|
||||
await expect(resolveAnnotationsById(assetA.id, '01JENTRYONASSETB123456')).rejects.toThrow(
|
||||
`Annotation "01JENTRYONASSETB123456" belongs to asset "${assetB.id}", not target asset "${assetA.id}".`,
|
||||
)
|
||||
})
|
||||
|
||||
it('throws an error if annotationId does not match any record', async () => {
|
||||
await expect(resolveAnnotationsById('asset-1', 'non_existent_id')).rejects.toThrow(
|
||||
'Annotation with ID "non_existent_id" not found.',
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import type { ShumaiMessageContext } from '@shumai/dtos'
|
||||
|
||||
export async function resolveAnnotationsById(
|
||||
assetId: string,
|
||||
annotationId?: string | null,
|
||||
): Promise<{ annotations: PrismaJson.AnnotationList | null; timestamp: number | null }> {
|
||||
if (!annotationId) {
|
||||
return { annotations: null, timestamp: null }
|
||||
}
|
||||
|
||||
// 1. Check AssetComment table (Comments Mode)
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: annotationId },
|
||||
select: { assetId: true, annotation: true, second: true },
|
||||
})
|
||||
if (comment) {
|
||||
if (comment.assetId !== assetId) {
|
||||
throw new Error(
|
||||
`Annotation "${annotationId}" belongs to asset "${comment.assetId}", not target asset "${assetId}".`,
|
||||
)
|
||||
}
|
||||
const list = comment.annotation
|
||||
const annotations =
|
||||
Array.isArray(list) && list.length > 0 ? (list as PrismaJson.AnnotationList) : null
|
||||
return { annotations, timestamp: comment.second }
|
||||
}
|
||||
|
||||
// 2. Check AgentSessionEntry table (1-on-1 Chat Mode)
|
||||
const entry = await prisma.agentSessionEntry.findUnique({
|
||||
where: { id: annotationId },
|
||||
select: { assetId: true, data: true },
|
||||
})
|
||||
if (entry) {
|
||||
const details = (entry.data as Record<string, unknown> | null)?.details as
|
||||
| ShumaiMessageContext
|
||||
| undefined
|
||||
const entryAssetId = entry.assetId || details?.currentAsset?.id
|
||||
if (entryAssetId && entryAssetId !== assetId) {
|
||||
throw new Error(
|
||||
`Annotation "${annotationId}" belongs to asset "${entryAssetId}", not target asset "${assetId}".`,
|
||||
)
|
||||
}
|
||||
const list = details?.annotations
|
||||
const annotations =
|
||||
Array.isArray(list) && list.length > 0 ? (list as unknown as PrismaJson.AnnotationList) : null
|
||||
const timestamp = details?.position?.type === 'time' ? details.position.seconds : null
|
||||
return { annotations, timestamp }
|
||||
}
|
||||
|
||||
throw new Error(`Annotation with ID "${annotationId}" not found.`)
|
||||
}
|
||||
@@ -3,7 +3,9 @@ import { createCreateFileTool } from './create-file'
|
||||
import { fieldsToTypeBoxSchema } from '../index'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { Value } from 'typebox/value'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
@@ -32,6 +34,12 @@ vi.mock('@shumai/core/src/asset/asset', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
auditLogService: {
|
||||
logAction: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('createCreateFileTool', () => {
|
||||
const createdFiles: string[] = []
|
||||
|
||||
@@ -86,7 +94,13 @@ describe('createCreateFileTool', () => {
|
||||
metadata: undefined,
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({ id: 'file-1', name: 'test', type: 'file', size: 100 })
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
id: 'file-1',
|
||||
name: 'test',
|
||||
type: 'file',
|
||||
size: 100,
|
||||
})
|
||||
})
|
||||
|
||||
it('should forward the metadata to assetService.createFile when provided', async () => {
|
||||
@@ -209,7 +223,13 @@ describe('createCreateFileTool', () => {
|
||||
metadata: undefined,
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({ id: 'file-1', name: 'test', type: 'file', size: 100 })
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
id: 'file-1',
|
||||
name: 'test',
|
||||
type: 'file',
|
||||
size: 100,
|
||||
})
|
||||
})
|
||||
|
||||
it('should default unknown extensions to text/plain when creating from content', async () => {
|
||||
@@ -326,4 +346,27 @@ describe('createCreateFileTool', () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- inspecting the runtime schema shape of the dynamically-built tool
|
||||
expect((tool.parameters as any).properties.metadata).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should pass agentId and record audit log when agentContext is provided', async () => {
|
||||
const filePath = createTempFile('# Agent file', 'agent-file.md')
|
||||
|
||||
const tool = createCreateFileTool('user-1', undefined, { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', { parent: 'folder-1', path: filePath, data: null })
|
||||
|
||||
expect(assetService.createFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
creatorId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
}),
|
||||
)
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.file_create,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: undefined,
|
||||
itemId: 'file-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { getFileMimeType, readFileMimeType } from '@shumai/core/src/utils/file-mime'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { type User } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { Type, type TSchema } from 'typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
@@ -26,7 +28,11 @@ function assertExactlyOneSource(params: CreateFileToolParams) {
|
||||
return hasPath ? params.path! : params.data!
|
||||
}
|
||||
|
||||
export function createCreateFileTool(userId: string, metadataSchema?: TSchema): AgentTool {
|
||||
export function createCreateFileTool(
|
||||
userId: string,
|
||||
metadataSchema?: TSchema,
|
||||
agentContext?: { teamId?: string; agentId?: string },
|
||||
): AgentTool {
|
||||
const createFileSchema = Type.Object(
|
||||
{
|
||||
parent: Type.String({
|
||||
@@ -121,9 +127,21 @@ export function createCreateFileTool(userId: string, metadataSchema?: TSchema):
|
||||
sizeByte: size,
|
||||
contentType: mimeType,
|
||||
creatorId: userId,
|
||||
agentId: agentContext?.agentId,
|
||||
metadata: p.metadata ?? undefined,
|
||||
})
|
||||
|
||||
if (agentContext?.teamId) {
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.file_create,
|
||||
teamId: agentContext.teamId,
|
||||
userId,
|
||||
agentId: agentContext.agentId,
|
||||
projectId: asset.projectId ?? undefined,
|
||||
itemId: asset.id,
|
||||
})
|
||||
}
|
||||
|
||||
const result = {
|
||||
id: asset.id,
|
||||
name: asset.name,
|
||||
@@ -133,7 +151,7 @@ export function createCreateFileTool(userId: string, metadataSchema?: TSchema):
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createCreateFolderTool } from './create-folder'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
@@ -22,6 +24,12 @@ vi.mock('@shumai/core/src/asset/asset', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
auditLogService: {
|
||||
logAction: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('createCreateFolderTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
@@ -43,13 +51,37 @@ describe('createCreateFolderTool', () => {
|
||||
parentId: 'root-folder',
|
||||
type: 'folder',
|
||||
creatorId: 'user-1',
|
||||
agentId: undefined,
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
id: 'folder-1',
|
||||
name: 'New Folder',
|
||||
type: 'folder',
|
||||
size: 0,
|
||||
})
|
||||
})
|
||||
|
||||
it('passes agentId and logs audit action when agentContext is provided', async () => {
|
||||
const tool = createCreateFolderTool('user-1', { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', { parent: 'root-folder', name: 'New Folder' })
|
||||
|
||||
expect(assetService.createAsset).toHaveBeenCalledWith({
|
||||
name: 'New Folder',
|
||||
parentId: 'root-folder',
|
||||
type: 'folder',
|
||||
creatorId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
})
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.folder_create,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: undefined,
|
||||
itemId: 'folder-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { type User } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
const createFolderSchema = Type.Object({
|
||||
parent: Type.String({
|
||||
@@ -14,7 +16,10 @@ const createFolderSchema = Type.Object({
|
||||
}),
|
||||
})
|
||||
|
||||
export function createCreateFolderTool(userId: string): AgentTool<typeof createFolderSchema> {
|
||||
export function createCreateFolderTool(
|
||||
userId: string,
|
||||
agentContext?: { teamId?: string; agentId?: string },
|
||||
): AgentTool<typeof createFolderSchema> {
|
||||
return {
|
||||
name: 'create_folder',
|
||||
label: 'Create Folder',
|
||||
@@ -33,8 +38,20 @@ export function createCreateFolderTool(userId: string): AgentTool<typeof createF
|
||||
parentId: params.parent,
|
||||
type: 'folder',
|
||||
creatorId: userId,
|
||||
agentId: agentContext?.agentId,
|
||||
})
|
||||
|
||||
if (agentContext?.teamId) {
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.folder_create,
|
||||
teamId: agentContext.teamId,
|
||||
userId,
|
||||
agentId: agentContext.agentId,
|
||||
projectId: newFolder.projectId ?? undefined,
|
||||
itemId: newFolder.id,
|
||||
})
|
||||
}
|
||||
|
||||
const result = {
|
||||
id: newFolder.id,
|
||||
name: newFolder.name,
|
||||
@@ -44,7 +61,7 @@ export function createCreateFolderTool(userId: string): AgentTool<typeof createF
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ import { createCreateVersionTool } from './create-version'
|
||||
import { fieldsToTypeBoxSchema } from '../index'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { Value } from 'typebox/value'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
@@ -31,6 +33,12 @@ vi.mock('@shumai/core/src/asset/asset', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
auditLogService: {
|
||||
logAction: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('createCreateVersionTool', () => {
|
||||
const createdFiles: string[] = []
|
||||
|
||||
@@ -83,7 +91,13 @@ describe('createCreateVersionTool', () => {
|
||||
metadata: undefined,
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({ id: 'file-1', name: 'v2', type: 'file', size: 100 })
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
id: 'file-1',
|
||||
name: 'v2',
|
||||
type: 'file',
|
||||
size: 100,
|
||||
})
|
||||
})
|
||||
|
||||
it('should forward the metadata to assetService.createVersion when provided', async () => {
|
||||
@@ -198,4 +212,30 @@ describe('createCreateVersionTool', () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- inspecting the runtime schema shape of the dynamically-built tool
|
||||
expect((tool.parameters as any).properties.metadata).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should pass agentId and record audit log when agentContext is provided', async () => {
|
||||
const filePath = createTempFile('# Agent version file', 'agent-ver-file.md')
|
||||
|
||||
const tool = createCreateVersionTool('user-1', undefined, {
|
||||
teamId: 'team-1',
|
||||
agentId: 'agent-1',
|
||||
})
|
||||
await tool.execute('call-1', { parent: 'file-1', path: filePath })
|
||||
|
||||
expect(assetService.createVersion).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
creatorId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
}),
|
||||
)
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.file_create,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: undefined,
|
||||
itemId: 'file-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { readFileMimeType } from '@shumai/core/src/utils/file-mime'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { type User } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
import { Type, type TSchema } from 'typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
@@ -16,7 +18,11 @@ interface CreateVersionToolParams {
|
||||
metadata?: Record<string, unknown> | null
|
||||
}
|
||||
|
||||
export function createCreateVersionTool(userId: string, metadataSchema?: TSchema): AgentTool {
|
||||
export function createCreateVersionTool(
|
||||
userId: string,
|
||||
metadataSchema?: TSchema,
|
||||
agentContext?: { teamId?: string; agentId?: string },
|
||||
): AgentTool {
|
||||
const createVersionSchema = Type.Object(
|
||||
{
|
||||
parent: Type.String({
|
||||
@@ -70,9 +76,21 @@ export function createCreateVersionTool(userId: string, metadataSchema?: TSchema
|
||||
sizeByte: fileSize,
|
||||
contentType: mimeType,
|
||||
creatorId: userId,
|
||||
agentId: agentContext?.agentId,
|
||||
metadata: p.metadata ?? undefined,
|
||||
})
|
||||
|
||||
if (agentContext?.teamId) {
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.file_create,
|
||||
teamId: agentContext.teamId,
|
||||
userId,
|
||||
agentId: agentContext.agentId,
|
||||
projectId: asset.projectId ?? undefined,
|
||||
itemId: asset.id,
|
||||
})
|
||||
}
|
||||
|
||||
const result = {
|
||||
id: asset.id,
|
||||
name: asset.name,
|
||||
@@ -82,7 +100,7 @@ export function createCreateVersionTool(userId: string, metadataSchema?: TSchema
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createDeleteAssetTool } from './delete-asset'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
Permission: { Edit: 'edit', Read: 'read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/asset/asset', () => ({
|
||||
assetService: {
|
||||
deleteAssets: vi.fn().mockResolvedValue(undefined),
|
||||
getAssetContext: vi.fn().mockResolvedValue({
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
auditLogService: {
|
||||
logAction: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/db', () => ({
|
||||
prisma: {
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
describe('createDeleteAssetTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('checks edit permission, deletes asset, and returns success for a file', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
type: 'file',
|
||||
projectId: 'proj-1',
|
||||
} as unknown as never)
|
||||
|
||||
const tool = createDeleteAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1' })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'edit',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(assetService.deleteAssets).toHaveBeenCalledWith(['asset-1'])
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
success: true,
|
||||
deletedAssetId: 'asset-1',
|
||||
})
|
||||
})
|
||||
|
||||
it('logs file_delete audit action when deleting a file with agentContext', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
type: 'file',
|
||||
projectId: 'proj-1',
|
||||
} as unknown as never)
|
||||
|
||||
const tool = createDeleteAssetTool('user-1', { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', { assetId: 'asset-1' })
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.file_delete,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: 'proj-1',
|
||||
itemId: 'asset-1',
|
||||
})
|
||||
})
|
||||
|
||||
it('logs folder_delete audit action when deleting a folder with agentContext', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'folder-1',
|
||||
type: 'folder',
|
||||
projectId: 'proj-1',
|
||||
} as unknown as never)
|
||||
|
||||
const tool = createDeleteAssetTool('user-1', { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', { assetId: 'folder-1' })
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.folder_delete,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: 'proj-1',
|
||||
itemId: 'folder-1',
|
||||
})
|
||||
})
|
||||
|
||||
it('throws when asset is not found', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
const tool = createDeleteAssetTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'missing-asset' })).rejects.toThrow(
|
||||
'Asset not found',
|
||||
)
|
||||
})
|
||||
|
||||
it('fails when permission check fails', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValueOnce(new Error('Permission denied'))
|
||||
const tool = createDeleteAssetTool('user-1')
|
||||
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1' })).rejects.toThrow(
|
||||
'Permission denied',
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,74 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { prisma, type User } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
const deleteAssetSchema = Type.Object(
|
||||
{
|
||||
assetId: Type.String({
|
||||
description:
|
||||
'The ID of the single asset (file or folder) to delete. This parameter is required.',
|
||||
}),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
)
|
||||
|
||||
export function createDeleteAssetTool(
|
||||
userId: string,
|
||||
agentContext?: { teamId?: string; agentId?: string },
|
||||
): AgentTool<typeof deleteAssetSchema> {
|
||||
return {
|
||||
name: 'delete_asset',
|
||||
label: 'Delete Asset',
|
||||
description:
|
||||
'Delete a single asset (file or folder) by its ID. Can only delete one asset at a time.',
|
||||
parameters: deleteAssetSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Asset,
|
||||
id: params.assetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: params.assetId },
|
||||
select: { id: true, type: true, projectId: true },
|
||||
})
|
||||
if (!asset) {
|
||||
throw new Error('Asset not found')
|
||||
}
|
||||
|
||||
const ctx = await assetService.getAssetContext(params.assetId).catch(() => ({
|
||||
teamId: agentContext?.teamId || '',
|
||||
projectId: asset.projectId ?? undefined,
|
||||
}))
|
||||
|
||||
await assetService.deleteAssets([params.assetId])
|
||||
|
||||
if (agentContext?.teamId) {
|
||||
await auditLogService.logAction({
|
||||
action: asset.type === 'folder' ? AuditAction.folder_delete : AuditAction.file_delete,
|
||||
teamId: agentContext.teamId || ctx.teamId,
|
||||
userId,
|
||||
agentId: agentContext.agentId,
|
||||
projectId: ctx.projectId ?? asset.projectId ?? undefined,
|
||||
itemId: params.assetId,
|
||||
})
|
||||
}
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
deletedAssetId: params.assetId,
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { createDownloadAssetTool } from './download-asset'
|
||||
import { prisma, type Asset } from '@shumai/db'
|
||||
import {
|
||||
createDownloadAssetTool,
|
||||
resolveAssetIdFromKey,
|
||||
resolveS3MediaBuffer,
|
||||
} from './download-asset'
|
||||
import { prisma, type Asset, type StorageKey } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import * as fs from 'fs'
|
||||
@@ -15,6 +19,10 @@ vi.mock('@shumai/db', async (importOriginal) => {
|
||||
findUnique: vi.fn(),
|
||||
findFirst: vi.fn(),
|
||||
},
|
||||
storageKey: {
|
||||
findFirst: vi.fn(),
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
@@ -52,163 +60,328 @@ describe('downloadAssetTool', () => {
|
||||
|
||||
it('should throw authorization error if user ID is empty', async () => {
|
||||
const tool = createDownloadAssetTool('')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1' })).rejects.toThrow(
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', key: null })).rejects.toThrow(
|
||||
'User ID is required for authorization.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if asset not found', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
it('should throw error if both assetId and key are null', async () => {
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1' })).rejects.toThrow(
|
||||
'Asset with ID asset-1 not found.',
|
||||
await expect(tool.execute('call-1', { assetId: null, key: null })).rejects.toThrow(
|
||||
'Provide exactly one of "assetId" (to download a workspace asset) or "key" (to download a specific S3 storage key). Set the unused parameter to null.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should download proxy image to .pi directory when imageTranscodes is present', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-img-1',
|
||||
name: 'sample_photo.jpg',
|
||||
storageKey: { key: 'raw/sample_photo.jpg' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/sample_photo.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-proxy-image-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
it('should throw error if both assetId and key are provided', async () => {
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-img-1' })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-img-1',
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/sample_photo.webp')
|
||||
|
||||
const expectedPath = path.join(piDir, 'asset-img-1_sample_photo.jpg')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(fs.readFileSync(expectedPath, 'utf-8')).toBe('fake-proxy-image-bytes')
|
||||
|
||||
expect(result.details.filePath).toBe(path.join('.pi', 'asset-img-1_sample_photo.jpg'))
|
||||
expect(result.details.contentType).toBe('image/webp')
|
||||
expect((result.content[0] as { type: 'text'; text: string }).text).toContain(
|
||||
'Remember to delete this temporary file when finished.',
|
||||
await expect(
|
||||
tool.execute('call-1', { assetId: 'asset-1', key: 'files/path/file.png' }),
|
||||
).rejects.toThrow(
|
||||
'Provide exactly one of "assetId" (to download a workspace asset) or "key" (to download a specific S3 storage key). Set the unused parameter to null.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should download proxy video to .pi directory when videoTranscodes is present', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-vid-1',
|
||||
name: 'video_test.mov',
|
||||
storageKey: { key: 'raw/video_test.mov' },
|
||||
media: {
|
||||
proxyType: 'video',
|
||||
videoTranscodes: [{ key: 'proxy/video_test.mp4' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
describe('Download by assetId', () => {
|
||||
it('should throw error if asset not found', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-video-bytes'),
|
||||
contentType: 'video/mp4',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-vid-1' })
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/video_test.mp4')
|
||||
|
||||
const expectedPath = path.join(piDir, 'asset-vid-1_video_test.mov')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(result.details.size).toBe(Buffer.from('fake-video-bytes').length)
|
||||
})
|
||||
|
||||
it('should download original file to .pi directory even when pdfTranscode is present', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-md-1',
|
||||
name: 'doc.md',
|
||||
storageKey: { key: 'raw/doc.md' },
|
||||
media: {
|
||||
proxyType: 'pdf',
|
||||
pdfTranscode: { key: 'proxy/doc.pdf' },
|
||||
original: { key: 'raw/doc.md' },
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('# fake markdown bytes'),
|
||||
contentType: 'text/markdown',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-md-1' })
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'raw/doc.md')
|
||||
|
||||
const expectedPath = path.join(piDir, 'asset-md-1_doc.md')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(result.details.contentType).toBe('text/markdown')
|
||||
})
|
||||
|
||||
it('should resolve version_stack asset to its latest version file', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'stack-1',
|
||||
name: '',
|
||||
type: 'version_stack',
|
||||
storageKey: null,
|
||||
media: null,
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.asset.findFirst).mockResolvedValue({
|
||||
id: 'file-ver-2',
|
||||
name: 'banana_pig.png',
|
||||
type: 'file',
|
||||
storageKey: { key: 'raw/banana_pig.png' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/banana_pig.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-version-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'stack-1' })
|
||||
|
||||
expect(prisma.asset.findFirst).toHaveBeenCalledWith({
|
||||
where: { parentId: 'stack-1', isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', key: null })).rejects.toThrow(
|
||||
'Asset with ID asset-1 not found.',
|
||||
)
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/banana_pig.webp')
|
||||
it('should download proxy image to .pi directory when downloading by assetId', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-img-1',
|
||||
name: 'sample_photo.jpg',
|
||||
storageKey: { key: 'raw/sample_photo.jpg' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/sample_photo.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const expectedPath = path.join(piDir, 'file-ver-2_banana_pig.png')
|
||||
createdFiles.push(expectedPath)
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-proxy-image-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(result.details.name).toBe('banana_pig.png')
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-img-1', key: null })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-img-1',
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/sample_photo.webp')
|
||||
|
||||
const expectedPath = path.join(piDir, 'asset-img-1_sample_photo.jpg')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(fs.readFileSync(expectedPath, 'utf-8')).toBe('fake-proxy-image-bytes')
|
||||
|
||||
expect(result.details.filePath).toBe(path.join('.pi', 'asset-img-1_sample_photo.jpg'))
|
||||
expect(result.details.contentType).toBe('image/webp')
|
||||
expect((result.content[0] as { type: 'text'; text: string }).text).toContain(
|
||||
'Remember to delete this temporary file when finished.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should resolve version_stack asset to its latest version file', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'stack-1',
|
||||
name: '',
|
||||
type: 'version_stack',
|
||||
storageKey: null,
|
||||
media: null,
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.asset.findFirst).mockResolvedValue({
|
||||
id: 'file-ver-2',
|
||||
name: 'banana_pig.png',
|
||||
type: 'file',
|
||||
storageKey: { key: 'raw/banana_pig.png' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/banana_pig.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-version-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'stack-1', key: null })
|
||||
|
||||
expect(prisma.asset.findFirst).toHaveBeenCalledWith({
|
||||
where: { parentId: 'stack-1', isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/banana_pig.webp')
|
||||
|
||||
const expectedPath = path.join(piDir, 'file-ver-2_banana_pig.png')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(result.details.name).toBe('banana_pig.png')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Download by S3 key and Authorization', () => {
|
||||
it('should reject keys that do not start with files/', async () => {
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', { assetId: null, key: 'avatars/user-1.png' }),
|
||||
).rejects.toThrow('Invalid storage key format: key must start with "files/"')
|
||||
})
|
||||
|
||||
it('should reject keys with path traversal (..)', async () => {
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', { assetId: null, key: 'files/../../etc/passwd' }),
|
||||
).rejects.toThrow('cannot contain ".."')
|
||||
})
|
||||
|
||||
it('should reject keys with insufficient path segments', async () => {
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: null, key: 'files/invalid' })).rejects.toThrow(
|
||||
'Invalid storage key format',
|
||||
)
|
||||
})
|
||||
|
||||
it('should reject keys that do not belong to any valid asset in database', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
vi.mocked(prisma.storageKey.findFirst).mockResolvedValue(null)
|
||||
vi.mocked(prisma.storageKey.findUnique).mockResolvedValue(null)
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', { assetId: null, key: 'files/nonexistent-id/shot.webp' }),
|
||||
).rejects.toThrow('does not belong to any valid asset or the asset has been deleted')
|
||||
})
|
||||
|
||||
it('should reject download when user lacks read permission on the owning asset (cross-project protection)', async () => {
|
||||
// Key belongs to asset-in-other-project
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-in-other-project',
|
||||
} as unknown as Asset)
|
||||
|
||||
// Authz rejects because user is in a different project/team
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValue(
|
||||
new Error('User does not have Read permission on asset asset-in-other-project'),
|
||||
)
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: null,
|
||||
key: 'files/asset-in-other-project/screenshots/secret_frame.webp',
|
||||
}),
|
||||
).rejects.toThrow('User does not have Read permission on asset asset-in-other-project')
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-in-other-project',
|
||||
})
|
||||
expect(s3Service.getObject).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should authorize and download derived artifact key (files/<assetId>/...)', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'ast-123',
|
||||
} as unknown as Asset)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-screenshot-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: null,
|
||||
key: 'files/ast-123/screenshots/shot_5.0s.webp',
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'ast-123',
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith(
|
||||
'shumai',
|
||||
'files/ast-123/screenshots/shot_5.0s.webp',
|
||||
)
|
||||
|
||||
const expectedPath = path.join(piDir, 'shot_5.0s.webp')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(fs.readFileSync(expectedPath, 'utf-8')).toBe('fake-screenshot-bytes')
|
||||
|
||||
expect(result.details.filePath).toBe(path.join('.pi', 'shot_5.0s.webp'))
|
||||
expect(result.details.key).toBe('files/ast-123/screenshots/shot_5.0s.webp')
|
||||
expect(result.details.assetId).toBe('ast-123')
|
||||
expect(result.details.size).toBe(Buffer.from('fake-screenshot-bytes').length)
|
||||
})
|
||||
|
||||
it('should authorize and download original upload key (files/<uploadUlid>/...) via StorageKey', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
vi.mocked(prisma.storageKey.findFirst).mockResolvedValue({
|
||||
id: 'sk-1',
|
||||
key: 'files/upload-ulid-456/photo.png',
|
||||
assets: [{ id: 'asset-photo-1' }],
|
||||
} as unknown as StorageKey & { assets: Asset[] })
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-original-photo-bytes'),
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: null,
|
||||
key: 'files/upload-ulid-456/photo.png',
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-photo-1',
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'files/upload-ulid-456/photo.png')
|
||||
|
||||
const expectedPath = path.join(piDir, 'photo.png')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(result.details.assetId).toBe('asset-photo-1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveAssetIdFromKey helper unit tests', () => {
|
||||
it('resolves directly by asset ID', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({ id: 'ast-999' } as unknown as Asset)
|
||||
const assetId = await resolveAssetIdFromKey('files/ast-999/pdf_pages/page_1.webp')
|
||||
expect(assetId).toBe('ast-999')
|
||||
})
|
||||
|
||||
it('resolves via StorageKey prefix', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
vi.mocked(prisma.storageKey.findFirst).mockResolvedValue({
|
||||
assets: [{ id: 'ast-888' }],
|
||||
} as unknown as StorageKey & { assets: Asset[] })
|
||||
const assetId = await resolveAssetIdFromKey('files/ulid-777/document.pdf')
|
||||
expect(assetId).toBe('ast-888')
|
||||
})
|
||||
|
||||
it('resolves via exact StorageKey match', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
vi.mocked(prisma.storageKey.findFirst).mockResolvedValue(null)
|
||||
vi.mocked(prisma.storageKey.findUnique).mockResolvedValue({
|
||||
assets: [{ id: 'ast-777' }],
|
||||
} as unknown as StorageKey & { assets: Asset[] })
|
||||
const assetId = await resolveAssetIdFromKey('files/ulid-special/document.pdf')
|
||||
expect(assetId).toBe('ast-777')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveS3MediaBuffer helper unit tests', () => {
|
||||
it('throws authorization error when userId is empty', async () => {
|
||||
await expect(resolveS3MediaBuffer('files/ast-1/img.png', '')).rejects.toThrow(
|
||||
'User ID is required for authorization.',
|
||||
)
|
||||
})
|
||||
|
||||
it('verifies authz permission on owning asset and returns buffer with mimeType', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({ id: 'ast-1' } as unknown as Asset)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('png-bytes'),
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const result = await resolveS3MediaBuffer('files/ast-1/image.png', 'user-123')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-123' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'ast-1',
|
||||
})
|
||||
expect(result.buffer.toString()).toBe('png-bytes')
|
||||
expect(result.mimeType).toBe('image/png')
|
||||
})
|
||||
|
||||
it('throws error when S3 file is not an image', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({ id: 'ast-1' } as unknown as Asset)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('some text'),
|
||||
contentType: 'text/plain',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(resolveS3MediaBuffer('files/ast-1/readme.txt', 'user-123')).rejects.toThrow(
|
||||
'has MIME type "text/plain", but an image is required.',
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -4,85 +4,144 @@ import { prisma, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
|
||||
const downloadAssetSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description:
|
||||
'The asset ID of the workspace asset to download to local disk. This parameter is required.',
|
||||
}),
|
||||
})
|
||||
export const downloadAssetSchema = Type.Object(
|
||||
{
|
||||
assetId: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'The asset ID of the workspace asset to download to local disk, or null if providing "key".',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
key: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'The storage S3 key of a specific media artifact (e.g. a screenshot or transcode key from read_asset) to download to local disk, or null if providing "assetId".',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
)
|
||||
|
||||
export async function resolveAssetIdFromKey(key: string): Promise<string> {
|
||||
if (!key.startsWith('files/') || key.includes('..')) {
|
||||
throw new Error(
|
||||
'Invalid storage key format: key must start with "files/" and cannot contain "..".',
|
||||
)
|
||||
}
|
||||
|
||||
const segments = key.split('/')
|
||||
if (segments.length < 3) {
|
||||
throw new Error(
|
||||
'Invalid storage key format: key must follow "files/<identifier>/<path>" structure.',
|
||||
)
|
||||
}
|
||||
|
||||
const identifier = segments[1]
|
||||
|
||||
// 1. Check if identifier is directly an Asset ID (Derived Artifacts: screenshots, pdf_pages, annotations, proxies)
|
||||
const assetById = await prisma.asset.findUnique({
|
||||
where: { id: identifier, isDeleted: false },
|
||||
select: { id: true },
|
||||
})
|
||||
if (assetById) {
|
||||
return assetById.id
|
||||
}
|
||||
|
||||
// 2. Check if identifier is a StorageKey directory prefix (Original uploads / agent-created files)
|
||||
const storageKeyByPrefix = await prisma.storageKey.findFirst({
|
||||
where: { key: { startsWith: `files/${identifier}/` } },
|
||||
include: {
|
||||
assets: {
|
||||
where: { isDeleted: false },
|
||||
select: { id: true },
|
||||
},
|
||||
},
|
||||
})
|
||||
if (storageKeyByPrefix?.assets && storageKeyByPrefix.assets.length > 0) {
|
||||
return storageKeyByPrefix.assets[0].id
|
||||
}
|
||||
|
||||
// 3. Exact StorageKey fallback
|
||||
const exactStorageKey = await prisma.storageKey.findUnique({
|
||||
where: { key },
|
||||
include: {
|
||||
assets: {
|
||||
where: { isDeleted: false },
|
||||
select: { id: true },
|
||||
},
|
||||
},
|
||||
})
|
||||
if (exactStorageKey?.assets && exactStorageKey.assets.length > 0) {
|
||||
return exactStorageKey.assets[0].id
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Storage key "${key}" does not belong to any valid asset or the asset has been deleted.`,
|
||||
)
|
||||
}
|
||||
|
||||
export async function resolveS3MediaBuffer(
|
||||
key: string,
|
||||
userId: string,
|
||||
): Promise<{ buffer: Buffer; mimeType: string }> {
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
const owningAssetId = await resolveAssetIdFromKey(key)
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: owningAssetId,
|
||||
})
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, key)
|
||||
const mimeType = getFileMimeType(
|
||||
buffer,
|
||||
key,
|
||||
contentType && contentType !== 'application/octet-stream'
|
||||
? contentType
|
||||
: 'application/octet-stream',
|
||||
)
|
||||
|
||||
if (!mimeType.startsWith('image/')) {
|
||||
throw new Error(`Storage key "${key}" has MIME type "${mimeType}", but an image is required.`)
|
||||
}
|
||||
|
||||
return { buffer, mimeType }
|
||||
}
|
||||
|
||||
export function createDownloadAssetTool(userId: string): AgentTool<typeof downloadAssetSchema> {
|
||||
return {
|
||||
name: 'download_asset',
|
||||
label: 'Download Asset',
|
||||
description:
|
||||
'Downloads a workspace asset (proxy version if available) to the local .pi directory for processing with local scripts or bash commands. Remember to delete the temporary downloaded file when finished.',
|
||||
'Downloads a workspace asset or specific storage S3 key to the local .pi directory for processing with local scripts or bash commands. ' +
|
||||
'Provide exactly one of "assetId" (for full assets) or "key" (for intermediate artifacts like screenshots from read_asset). ' +
|
||||
'Remember to delete the temporary downloaded file when finished.',
|
||||
parameters: downloadAssetSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const assetId = params.assetId
|
||||
const { assetId, key } = params
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
const hasAssetId = assetId !== null
|
||||
const hasKey = key !== null
|
||||
|
||||
let asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
if (asset.type === 'version_stack') {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (latestVersion) {
|
||||
asset = latestVersion
|
||||
}
|
||||
}
|
||||
|
||||
let mediaKey = asset.storageKey?.key
|
||||
|
||||
if (asset.media) {
|
||||
const mediaInfo = asset.media as unknown as PrismaJson.MediaInfo
|
||||
if (
|
||||
mediaInfo.proxyType === 'image' ||
|
||||
(mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0)
|
||||
) {
|
||||
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
|
||||
}
|
||||
} else if (
|
||||
mediaInfo.proxyType === 'video' ||
|
||||
(mediaInfo.videoTranscodes && mediaInfo.videoTranscodes.length > 0) ||
|
||||
mediaInfo.videoPreview?.key
|
||||
) {
|
||||
if (mediaInfo.videoTranscodes && mediaInfo.videoTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.videoTranscodes[0].key || mediaKey
|
||||
} else if (mediaInfo.videoPreview?.key) {
|
||||
mediaKey = mediaInfo.videoPreview.key
|
||||
}
|
||||
} else if (mediaInfo.original?.key) {
|
||||
mediaKey = mediaInfo.original.key || mediaKey
|
||||
}
|
||||
}
|
||||
|
||||
if (!mediaKey) {
|
||||
throw new Error(`No media content found for asset ${assetId}.`)
|
||||
if (hasAssetId === hasKey) {
|
||||
throw new Error(
|
||||
'Provide exactly one of "assetId" (to download a workspace asset) or "key" (to download a specific S3 storage key). Set the unused parameter to null.',
|
||||
)
|
||||
}
|
||||
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
@@ -90,32 +149,132 @@ export function createDownloadAssetTool(userId: string): AgentTool<typeof downlo
|
||||
fs.mkdirSync(piDir, { recursive: true })
|
||||
}
|
||||
|
||||
const safeName = sanitizeFilename(asset.name || 'file')
|
||||
const filename = `${asset.id}_${safeName}`
|
||||
const targetFilePath = path.join(piDir, filename)
|
||||
const relativePath = path.join('.pi', filename)
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, mediaKey)
|
||||
|
||||
fs.writeFileSync(targetFilePath, buffer)
|
||||
// Branch A: Download by assetId
|
||||
if (assetId) {
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Downloaded asset "${asset.name}" (ID: ${assetId}) to "${relativePath}". Remember to delete this temporary file when finished.`,
|
||||
let asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
if (asset.type === 'version_stack') {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (latestVersion) {
|
||||
asset = latestVersion
|
||||
}
|
||||
}
|
||||
|
||||
let mediaKey = asset.storageKey?.key
|
||||
|
||||
if (asset.media) {
|
||||
const mediaInfo = asset.media as unknown as PrismaJson.MediaInfo
|
||||
if (
|
||||
mediaInfo.proxyType === 'image' ||
|
||||
(mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0)
|
||||
) {
|
||||
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
|
||||
}
|
||||
} else if (
|
||||
mediaInfo.proxyType === 'video' ||
|
||||
(mediaInfo.videoTranscodes && mediaInfo.videoTranscodes.length > 0) ||
|
||||
mediaInfo.videoPreview?.key
|
||||
) {
|
||||
if (mediaInfo.videoTranscodes && mediaInfo.videoTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.videoTranscodes[0].key || mediaKey
|
||||
} else if (mediaInfo.videoPreview?.key) {
|
||||
mediaKey = mediaInfo.videoPreview.key
|
||||
}
|
||||
} else if (mediaInfo.original?.key) {
|
||||
mediaKey = mediaInfo.original.key || mediaKey
|
||||
}
|
||||
}
|
||||
|
||||
if (!mediaKey) {
|
||||
throw new Error(`No media content found for asset ${assetId}.`)
|
||||
}
|
||||
|
||||
const safeName = sanitizeFilename(asset.name || 'file')
|
||||
const filename = `${asset.id}_${safeName}`
|
||||
const targetFilePath = path.join(piDir, filename)
|
||||
const relativePath = path.join('.pi', filename)
|
||||
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, mediaKey)
|
||||
fs.writeFileSync(targetFilePath, buffer)
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Downloaded asset "${asset.name}" (ID: ${assetId}) to "${relativePath}". Remember to delete this temporary file when finished.`,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
assetId,
|
||||
name: asset.name,
|
||||
filePath: relativePath,
|
||||
absolutePath: targetFilePath,
|
||||
contentType: contentType || 'application/octet-stream',
|
||||
size: buffer.length,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
assetId,
|
||||
name: asset.name,
|
||||
filePath: relativePath,
|
||||
absolutePath: targetFilePath,
|
||||
contentType: contentType || 'application/octet-stream',
|
||||
size: buffer.length,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Branch B: Download by S3 storage key
|
||||
if (key) {
|
||||
const owningAssetId = await resolveAssetIdFromKey(key)
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: owningAssetId,
|
||||
})
|
||||
|
||||
const rawBasename = path.basename(key)
|
||||
const safeBasename = sanitizeFilename(rawBasename) || 'downloaded_file'
|
||||
const filename = safeBasename
|
||||
const targetFilePath = path.join(piDir, filename)
|
||||
const relativePath = path.join('.pi', filename)
|
||||
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, key)
|
||||
fs.writeFileSync(targetFilePath, buffer)
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Downloaded storage key "${key}" to "${relativePath}". Remember to delete this temporary file when finished.`,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
key,
|
||||
assetId: owningAssetId,
|
||||
filePath: relativePath,
|
||||
absolutePath: targetFilePath,
|
||||
contentType: contentType || 'application/octet-stream',
|
||||
size: buffer.length,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Invalid download_asset parameters.')
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
import { vi } from 'vitest'
|
||||
|
||||
vi.mock('fs')
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { createGenerateImageTool } from './generate-image'
|
||||
import * as downloadAsset from './download-asset'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { EnabledMediaModel } from '@shumai/dtos'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
|
||||
describe('generate_image tool', () => {
|
||||
const enabledModels: EnabledMediaModel[] = [
|
||||
{
|
||||
id: 'em-1',
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3 (OpenAI)',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: 'em-2',
|
||||
type: 'image',
|
||||
provider: 'azure',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3 (Azure)',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: 'em-3',
|
||||
type: 'image',
|
||||
provider: 'google',
|
||||
modelId: 'imagen-3.0-generate-002',
|
||||
name: 'Imagen 3',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
]
|
||||
|
||||
const providerKeys = {
|
||||
openai: { apiKey: 'mock-openai-key' },
|
||||
azure: { apiKey: 'mock-azure-key' },
|
||||
google: { apiKey: 'mock-google-key' },
|
||||
}
|
||||
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
|
||||
vi.spyOn(fs, 'mkdirSync').mockImplementation(() => undefined)
|
||||
vi.spyOn(fs, 'writeFileSync').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('creates tool with correct metadata and provider-prefixed model schema', () => {
|
||||
const tool = createGenerateImageTool(enabledModels, providerKeys, 'user-1')
|
||||
expect(tool.name).toBe('generate_image')
|
||||
expect(tool.label).toBe('Generate Image')
|
||||
expect(tool.description).toContain('Generate an image')
|
||||
const modelDesc = (tool.parameters as { properties: { model: { description: string } } })
|
||||
.properties.model.description
|
||||
expect(modelDesc).toContain('openai:dall-e-3')
|
||||
expect(modelDesc).toContain('azure:dall-e-3')
|
||||
})
|
||||
|
||||
it('generates an image successfully and writes to .pi directory', async () => {
|
||||
const tool = createGenerateImageTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const mockBuffer = Buffer.from('fake-image-png-content')
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateImage').mockResolvedValue({
|
||||
buffer: mockBuffer,
|
||||
mimeType: 'image/png',
|
||||
})
|
||||
const writeSpy = vi.spyOn(fs, 'writeFileSync').mockImplementation(() => {})
|
||||
|
||||
const result = await tool.execute(
|
||||
'call-1',
|
||||
{
|
||||
prompt: 'A futuristic city skyline at dusk',
|
||||
model: 'openai:dall-e-3',
|
||||
aspectRatio: '16:9',
|
||||
size: null,
|
||||
seed: 42,
|
||||
images: null,
|
||||
mask: null,
|
||||
outputFileName: 'test_city.png',
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
apiKey: 'mock-openai-key',
|
||||
prompt: 'A futuristic city skyline at dusk',
|
||||
aspectRatio: '16:9',
|
||||
seed: 42,
|
||||
}),
|
||||
)
|
||||
|
||||
const expectedPath = path.join(piDir, 'test_city.png')
|
||||
expect(writeSpy).toHaveBeenCalledWith(expectedPath, mockBuffer)
|
||||
|
||||
const textContent = result.content[0]
|
||||
expect(textContent.type).toBe('text')
|
||||
if (textContent.type === 'text') {
|
||||
expect(textContent.text).toContain('Successfully generated image')
|
||||
expect(textContent.text).toContain('.pi/test_city.png')
|
||||
}
|
||||
expect(result.details).toMatchObject({
|
||||
fileName: 'test_city.png',
|
||||
provider: 'openai',
|
||||
model: 'dall-e-3',
|
||||
})
|
||||
})
|
||||
|
||||
it('disambiguates identical model IDs across different providers', async () => {
|
||||
const tool = createGenerateImageTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateImage').mockResolvedValue({
|
||||
buffer: Buffer.from('image-bytes'),
|
||||
mimeType: 'image/png',
|
||||
})
|
||||
|
||||
// Invoke azure:dall-e-3
|
||||
await tool.execute(
|
||||
'call-azure',
|
||||
{
|
||||
prompt: 'Azure generation',
|
||||
model: 'azure:dall-e-3',
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
seed: null,
|
||||
images: null,
|
||||
mask: null,
|
||||
outputFileName: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'azure',
|
||||
modelId: 'dall-e-3',
|
||||
apiKey: 'mock-azure-key',
|
||||
}),
|
||||
)
|
||||
|
||||
// Invoke openai:dall-e-3
|
||||
await tool.execute(
|
||||
'call-openai',
|
||||
{
|
||||
prompt: 'OpenAI generation',
|
||||
model: 'openai:dall-e-3',
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
seed: null,
|
||||
images: null,
|
||||
mask: null,
|
||||
outputFileName: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
apiKey: 'mock-openai-key',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('defaults to first enabled model when model is null', async () => {
|
||||
const tool = createGenerateImageTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const mockBuffer = Buffer.from('fake-image-bytes')
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateImage').mockResolvedValue({
|
||||
buffer: mockBuffer,
|
||||
mimeType: 'image/png',
|
||||
})
|
||||
|
||||
await tool.execute(
|
||||
'call-2',
|
||||
{
|
||||
prompt: 'A cute cat',
|
||||
model: null,
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
seed: null,
|
||||
images: null,
|
||||
mask: null,
|
||||
outputFileName: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('resolves reference images and mask via resolveS3MediaBuffer', async () => {
|
||||
const tool = createGenerateImageTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateImage').mockResolvedValue({
|
||||
buffer: Buffer.from('out'),
|
||||
mimeType: 'image/jpeg',
|
||||
})
|
||||
|
||||
const resolverSpy = vi
|
||||
.spyOn(downloadAsset, 'resolveS3MediaBuffer')
|
||||
.mockImplementation(async (key: string) => ({
|
||||
buffer: Buffer.from(`data-for-${key}`),
|
||||
mimeType: 'image/png',
|
||||
}))
|
||||
|
||||
await tool.execute(
|
||||
'call-3',
|
||||
{
|
||||
prompt: 'Transform into watercolor',
|
||||
model: 'openai:dall-e-3',
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
seed: null,
|
||||
images: ['files/ast-1/ref1.png', 'files/ast-2/ref2.png'],
|
||||
mask: 'files/ast-mask/mask.png',
|
||||
outputFileName: 'test_watercolor.jpg',
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(resolverSpy).toHaveBeenCalledWith('files/ast-1/ref1.png', 'user-1')
|
||||
expect(resolverSpy).toHaveBeenCalledWith('files/ast-2/ref2.png', 'user-1')
|
||||
expect(resolverSpy).toHaveBeenCalledWith('files/ast-mask/mask.png', 'user-1')
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
images: [
|
||||
Buffer.from('data-for-files/ast-1/ref1.png'),
|
||||
Buffer.from('data-for-files/ast-2/ref2.png'),
|
||||
],
|
||||
mask: Buffer.from('data-for-files/ast-mask/mask.png'),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('fails gracefully when API key is missing', async () => {
|
||||
const tool = createGenerateImageTool(enabledModels, {}, 'user-1') // No keys configured
|
||||
|
||||
await expect(
|
||||
tool.execute(
|
||||
'call-4',
|
||||
{
|
||||
prompt: 'A sunset',
|
||||
model: 'openai:dall-e-3',
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
seed: null,
|
||||
images: null,
|
||||
mask: null,
|
||||
outputFileName: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
),
|
||||
).rejects.toThrow('API key for provider "openai" is not configured')
|
||||
})
|
||||
|
||||
it('fails gracefully when requested model is not enabled', async () => {
|
||||
const tool = createGenerateImageTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
await expect(
|
||||
tool.execute(
|
||||
'call-5',
|
||||
{
|
||||
prompt: 'A sunset',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
model: 'non-existent-model' as any,
|
||||
aspectRatio: null,
|
||||
size: null,
|
||||
seed: null,
|
||||
images: null,
|
||||
mask: null,
|
||||
outputFileName: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
),
|
||||
).rejects.toThrow('Image model "non-existent-model" is not enabled')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,182 @@
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { EnabledMediaModel } from '@shumai/dtos'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { Type, type Static, type TSchema } from 'typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { ulid } from 'ulid'
|
||||
import { resolveS3MediaBuffer } from './download-asset'
|
||||
|
||||
export function createGenerateImageTool(
|
||||
enabledModels: EnabledMediaModel[],
|
||||
providerKeys: Record<string, { apiKey?: string }>,
|
||||
userId?: string,
|
||||
): AgentTool {
|
||||
const modelIdentifiers = enabledModels.map((m) => `${m.provider}:${m.modelId}`)
|
||||
const modelSchemas =
|
||||
modelIdentifiers.length === 1
|
||||
? Type.Literal(modelIdentifiers[0])
|
||||
: Type.Union(
|
||||
modelIdentifiers.map((id) => Type.Literal(id)) as unknown as [
|
||||
TSchema,
|
||||
TSchema,
|
||||
...TSchema[],
|
||||
],
|
||||
)
|
||||
|
||||
const generateImageSchema = Type.Object(
|
||||
{
|
||||
prompt: Type.String({
|
||||
description: 'The text prompt describing the image to generate in detail.',
|
||||
}),
|
||||
model: Type.Union([modelSchemas, Type.Null()], {
|
||||
description: `The image model to use, in "provider:modelId" format. Available enabled models: ${modelIdentifiers.join(', ')}. Pass null to use the first enabled model (${modelIdentifiers[0]}).`,
|
||||
}),
|
||||
outputFileName: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Optional desired filename for the generated image (e.g. "cover.png"). Set to null for auto-generated name.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
images: Type.Union([
|
||||
Type.Array(Type.String(), {
|
||||
description:
|
||||
'Optional list of storage S3 keys of images to use as image-to-image or style reference. Call read_asset with s3KeyOnly: true to obtain the S3 key of an asset or video frame without loading its content into context. Set to null if not using.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
mask: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Optional storage S3 key of an inpainting mask image (white=inpaint, black=keep). Call read_asset with s3KeyOnly: true to obtain the S3 key. Set to null if not inpainting.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
aspectRatio: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Optional aspect ratio, e.g. "1:1", "16:9", "9:16", "4:3", "3:4", "2:3", "3:2". Set to null for default.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
size: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Optional target image dimensions, e.g. "1024x1024", "1536x1024". Set to null for default.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
seed: Type.Union([
|
||||
Type.Integer({
|
||||
description:
|
||||
'Optional random seed for reproducible generations. Set to null for random generation.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
)
|
||||
|
||||
type GenerateImageParams = Static<typeof generateImageSchema>
|
||||
|
||||
return {
|
||||
name: 'generate_image',
|
||||
label: 'Generate Image',
|
||||
description:
|
||||
'Generate an image using a configured image model. Supports text-to-image, reference images (image-to-image using S3 keys from read_asset), and inpainting (using a mask image S3 key from read_asset). The generated file is saved in the local .pi/ directory.',
|
||||
parameters: generateImageSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const p = params as GenerateImageParams
|
||||
|
||||
const defaultModel = `${enabledModels[0].provider}:${enabledModels[0].modelId}`
|
||||
const selected = p.model || defaultModel
|
||||
const modelConfig =
|
||||
enabledModels.find((m) => `${m.provider}:${m.modelId}` === selected) ||
|
||||
enabledModels.find((m) => m.modelId === selected)
|
||||
|
||||
if (!modelConfig) {
|
||||
throw new Error(
|
||||
`Image model "${selected}" is not enabled. Available enabled models: ${modelIdentifiers.join(', ')}`,
|
||||
)
|
||||
}
|
||||
|
||||
const customKey = providerKeys[modelConfig.provider]?.apiKey
|
||||
const apiKey = mediaGenerationService.resolveApiKey(modelConfig.provider, customKey)
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
`API key for provider "${modelConfig.provider}" is not configured. Please configure it in settings.`,
|
||||
)
|
||||
}
|
||||
|
||||
let resolvedImages: Buffer[] | undefined
|
||||
if (p.images && Array.isArray(p.images) && p.images.length > 0) {
|
||||
resolvedImages = await Promise.all(
|
||||
p.images.map(async (key: string) => {
|
||||
const { buffer } = await resolveS3MediaBuffer(key, userId || '')
|
||||
return buffer
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
let resolvedMask: Buffer | undefined
|
||||
if (p.mask) {
|
||||
const { buffer } = await resolveS3MediaBuffer(p.mask, userId || '')
|
||||
resolvedMask = buffer
|
||||
}
|
||||
|
||||
const { buffer, mimeType } = await mediaGenerationService.generateImage({
|
||||
provider: modelConfig.provider,
|
||||
modelId: modelConfig.modelId,
|
||||
apiKey,
|
||||
prompt: p.prompt,
|
||||
images: resolvedImages,
|
||||
mask: resolvedMask,
|
||||
aspectRatio: (p.aspectRatio as `${number}:${number}`) || undefined,
|
||||
size: (p.size as `${number}x${number}`) || undefined,
|
||||
seed: p.seed !== null && p.seed !== undefined ? p.seed : undefined,
|
||||
})
|
||||
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
if (!fs.existsSync(piDir)) {
|
||||
fs.mkdirSync(piDir, { recursive: true })
|
||||
}
|
||||
|
||||
const safeMime = mimeType || 'image/png'
|
||||
let ext = '.png'
|
||||
if (safeMime.includes('jpeg') || safeMime.includes('jpg')) ext = '.jpg'
|
||||
else if (safeMime.includes('webp')) ext = '.webp'
|
||||
|
||||
let filename: string
|
||||
if (p.outputFileName) {
|
||||
const safe = sanitizeFilename(p.outputFileName)
|
||||
filename = safe.includes('.') ? safe : `${safe}${ext}`
|
||||
} else {
|
||||
filename = `generated_image_${ulid()}${ext}`
|
||||
}
|
||||
|
||||
const targetPath = path.join(piDir, filename)
|
||||
const relativePath = path.join('.pi', filename)
|
||||
fs.writeFileSync(targetPath, buffer)
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Successfully generated image using model "${modelConfig.modelId}" (${modelConfig.provider}). Saved to "${relativePath}". Size: ${(buffer.length / 1024).toFixed(1)} KB.`,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
fileName: filename,
|
||||
filePath: relativePath,
|
||||
absolutePath: targetPath,
|
||||
size: buffer.length,
|
||||
mimeType,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.modelId,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
import { vi } from 'vitest'
|
||||
|
||||
vi.mock('fs')
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { createGenerateVideoTool } from './generate-video'
|
||||
import * as downloadAsset from './download-asset'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { EnabledMediaModel } from '@shumai/dtos'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
|
||||
describe('generate_video tool', () => {
|
||||
const enabledModels: EnabledMediaModel[] = [
|
||||
{
|
||||
id: 'vm-1',
|
||||
type: 'video',
|
||||
provider: 'klingai',
|
||||
modelId: 'kling-v1-standard',
|
||||
name: 'Kling v1 Standard',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: 'vm-2',
|
||||
type: 'video',
|
||||
provider: 'fal',
|
||||
modelId: 'kling-v1-standard',
|
||||
name: 'Kling v1 Standard (fal)',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: 'vm-3',
|
||||
type: 'video',
|
||||
provider: 'google',
|
||||
modelId: 'veo-2.0-generate-001',
|
||||
name: 'Veo 2',
|
||||
createdAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
]
|
||||
|
||||
const providerKeys = {
|
||||
klingai: { apiKey: 'mock-kling-key' },
|
||||
fal: { apiKey: 'mock-fal-key' },
|
||||
google: { apiKey: 'mock-google-key' },
|
||||
}
|
||||
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
vi.spyOn(fs, 'existsSync').mockReturnValue(true)
|
||||
vi.spyOn(fs, 'mkdirSync').mockImplementation(() => undefined)
|
||||
vi.spyOn(fs, 'writeFileSync').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('creates tool with correct metadata and provider-prefixed model schema', () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, providerKeys, 'user-1')
|
||||
expect(tool.name).toBe('generate_video')
|
||||
expect(tool.label).toBe('Generate Video')
|
||||
expect(tool.description).toContain('Generate a video')
|
||||
const modelDesc = (tool.parameters as { properties: { model: { description: string } } })
|
||||
.properties.model.description
|
||||
expect(modelDesc).toContain('klingai:kling-v1-standard')
|
||||
expect(modelDesc).toContain('fal:kling-v1-standard')
|
||||
})
|
||||
|
||||
it('generates text_to_video successfully', async () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const mockBuffer = Buffer.from('fake-mp4-video-stream')
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateVideo').mockResolvedValue({
|
||||
buffer: mockBuffer,
|
||||
mimeType: 'video/mp4',
|
||||
})
|
||||
const writeSpy = vi.spyOn(fs, 'writeFileSync').mockImplementation(() => {})
|
||||
|
||||
const result = await tool.execute(
|
||||
'call-vid-1',
|
||||
{
|
||||
mode: 'text_to_video',
|
||||
model: 'klingai:kling-v1-standard',
|
||||
outputFileName: 'test_nature.mp4',
|
||||
textToVideoConfig: {
|
||||
prompt: 'A gentle waterfall in a lush forest with cinematic lighting',
|
||||
},
|
||||
imageToVideoConfig: null,
|
||||
firstLastFrameConfig: null,
|
||||
referenceToVideoConfig: null,
|
||||
aspectRatio: '16:9',
|
||||
resolution: null,
|
||||
duration: 5,
|
||||
fps: 24,
|
||||
generateAudio: null,
|
||||
seed: 123,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'klingai',
|
||||
modelId: 'kling-v1-standard',
|
||||
apiKey: 'mock-kling-key',
|
||||
mode: 'text_to_video',
|
||||
prompt: 'A gentle waterfall in a lush forest with cinematic lighting',
|
||||
aspectRatio: '16:9',
|
||||
duration: 5,
|
||||
fps: 24,
|
||||
seed: 123,
|
||||
}),
|
||||
)
|
||||
|
||||
const expectedPath = path.join(piDir, 'test_nature.mp4')
|
||||
expect(writeSpy).toHaveBeenCalledWith(expectedPath, mockBuffer)
|
||||
|
||||
const textContent = result.content[0]
|
||||
expect(textContent.type).toBe('text')
|
||||
if (textContent.type === 'text') {
|
||||
expect(textContent.text).toContain('Successfully generated video')
|
||||
expect(textContent.text).toContain('.pi/test_nature.mp4')
|
||||
}
|
||||
expect(result.details).toMatchObject({
|
||||
fileName: 'test_nature.mp4',
|
||||
provider: 'klingai',
|
||||
model: 'kling-v1-standard',
|
||||
mode: 'text_to_video',
|
||||
})
|
||||
})
|
||||
|
||||
it('disambiguates identical model IDs across different providers', async () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateVideo').mockResolvedValue({
|
||||
buffer: Buffer.from('video-bytes'),
|
||||
mimeType: 'video/mp4',
|
||||
})
|
||||
|
||||
// Select fal:kling-v1-standard
|
||||
await tool.execute(
|
||||
'call-fal',
|
||||
{
|
||||
mode: 'text_to_video',
|
||||
model: 'fal:kling-v1-standard',
|
||||
outputFileName: null,
|
||||
textToVideoConfig: { prompt: 'A soaring eagle' },
|
||||
imageToVideoConfig: null,
|
||||
firstLastFrameConfig: null,
|
||||
referenceToVideoConfig: null,
|
||||
aspectRatio: null,
|
||||
resolution: null,
|
||||
duration: null,
|
||||
fps: null,
|
||||
generateAudio: null,
|
||||
seed: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'fal',
|
||||
modelId: 'kling-v1-standard',
|
||||
apiKey: 'mock-fal-key',
|
||||
}),
|
||||
)
|
||||
|
||||
// Select klingai:kling-v1-standard
|
||||
await tool.execute(
|
||||
'call-klingai',
|
||||
{
|
||||
mode: 'text_to_video',
|
||||
model: 'klingai:kling-v1-standard',
|
||||
outputFileName: null,
|
||||
textToVideoConfig: { prompt: 'A roaring tiger' },
|
||||
imageToVideoConfig: null,
|
||||
firstLastFrameConfig: null,
|
||||
referenceToVideoConfig: null,
|
||||
aspectRatio: null,
|
||||
resolution: null,
|
||||
duration: null,
|
||||
fps: null,
|
||||
generateAudio: null,
|
||||
seed: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
provider: 'klingai',
|
||||
modelId: 'kling-v1-standard',
|
||||
apiKey: 'mock-kling-key',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('generates image_to_video using resolveS3MediaBuffer', async () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateVideo').mockResolvedValue({
|
||||
buffer: Buffer.from('video-bytes'),
|
||||
mimeType: 'video/mp4',
|
||||
})
|
||||
|
||||
const resolverSpy = vi.spyOn(downloadAsset, 'resolveS3MediaBuffer').mockResolvedValue({
|
||||
buffer: Buffer.from('resolved-image-data'),
|
||||
mimeType: 'image/png',
|
||||
})
|
||||
|
||||
await tool.execute(
|
||||
'call-vid-2',
|
||||
{
|
||||
mode: 'image_to_video',
|
||||
model: 'klingai:kling-v1-standard',
|
||||
outputFileName: 'animate_cat.mp4',
|
||||
textToVideoConfig: null,
|
||||
imageToVideoConfig: {
|
||||
image: 'files/ast-1/img.png',
|
||||
prompt: 'Cat starts blinking and looks to the side',
|
||||
},
|
||||
firstLastFrameConfig: null,
|
||||
referenceToVideoConfig: null,
|
||||
aspectRatio: null,
|
||||
resolution: null,
|
||||
duration: null,
|
||||
fps: null,
|
||||
generateAudio: null,
|
||||
seed: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(resolverSpy).toHaveBeenCalledWith('files/ast-1/img.png', 'user-1')
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mode: 'image_to_video',
|
||||
image: Buffer.from('resolved-image-data'),
|
||||
prompt: 'Cat starts blinking and looks to the side',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('generates first_last_frame using resolveS3MediaBuffer', async () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateVideo').mockResolvedValue({
|
||||
buffer: Buffer.from('transition-video'),
|
||||
mimeType: 'video/mp4',
|
||||
})
|
||||
|
||||
vi.spyOn(downloadAsset, 'resolveS3MediaBuffer').mockImplementation(async (key: string) => ({
|
||||
buffer: Buffer.from(`frame-data-${key}`),
|
||||
mimeType: 'image/png',
|
||||
}))
|
||||
|
||||
await tool.execute(
|
||||
'call-vid-3',
|
||||
{
|
||||
mode: 'first_last_frame',
|
||||
model: 'klingai:kling-v1-standard',
|
||||
outputFileName: 'transition.mp4',
|
||||
textToVideoConfig: null,
|
||||
imageToVideoConfig: null,
|
||||
firstLastFrameConfig: {
|
||||
firstFrame: 'files/ast-1/start.png',
|
||||
lastFrame: 'files/ast-2/end.png',
|
||||
prompt: 'Smooth morph from sunrise to night',
|
||||
},
|
||||
referenceToVideoConfig: null,
|
||||
aspectRatio: null,
|
||||
resolution: null,
|
||||
duration: null,
|
||||
fps: null,
|
||||
generateAudio: null,
|
||||
seed: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mode: 'first_last_frame',
|
||||
firstFrame: Buffer.from('frame-data-files/ast-1/start.png'),
|
||||
lastFrame: Buffer.from('frame-data-files/ast-2/end.png'),
|
||||
prompt: 'Smooth morph from sunrise to night',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('generates reference_to_video using resolveS3MediaBuffer', async () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, providerKeys, 'user-1')
|
||||
|
||||
const generateSpy = vi.spyOn(mediaGenerationService, 'generateVideo').mockResolvedValue({
|
||||
buffer: Buffer.from('ref-video'),
|
||||
mimeType: 'video/mp4',
|
||||
})
|
||||
|
||||
vi.spyOn(downloadAsset, 'resolveS3MediaBuffer').mockImplementation(async (key: string) => ({
|
||||
buffer: Buffer.from(`ref-data-${key}`),
|
||||
mimeType: 'image/png',
|
||||
}))
|
||||
|
||||
await tool.execute(
|
||||
'call-vid-4',
|
||||
{
|
||||
mode: 'reference_to_video',
|
||||
model: 'klingai:kling-v1-standard',
|
||||
outputFileName: 'reference_output.mp4',
|
||||
textToVideoConfig: null,
|
||||
imageToVideoConfig: null,
|
||||
firstLastFrameConfig: null,
|
||||
referenceToVideoConfig: {
|
||||
references: ['files/ast-1/ref1.png', 'files/ast-2/ref2.png'],
|
||||
prompt: 'Generate an anime dance following these characters',
|
||||
},
|
||||
aspectRatio: null,
|
||||
resolution: null,
|
||||
duration: null,
|
||||
fps: null,
|
||||
generateAudio: null,
|
||||
seed: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
)
|
||||
|
||||
expect(generateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mode: 'reference_to_video',
|
||||
inputReferences: [
|
||||
Buffer.from('ref-data-files/ast-1/ref1.png'),
|
||||
Buffer.from('ref-data-files/ast-2/ref2.png'),
|
||||
],
|
||||
prompt: 'Generate an anime dance following these characters',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('fails gracefully when API key is missing', async () => {
|
||||
const tool = createGenerateVideoTool(enabledModels, {}, 'user-1')
|
||||
|
||||
await expect(
|
||||
tool.execute(
|
||||
'call-vid-5',
|
||||
{
|
||||
mode: 'text_to_video',
|
||||
model: 'klingai:kling-v1-standard',
|
||||
outputFileName: null,
|
||||
textToVideoConfig: { prompt: 'A simple video' },
|
||||
imageToVideoConfig: null,
|
||||
firstLastFrameConfig: null,
|
||||
referenceToVideoConfig: null,
|
||||
aspectRatio: null,
|
||||
resolution: null,
|
||||
duration: null,
|
||||
fps: null,
|
||||
generateAudio: null,
|
||||
seed: null,
|
||||
},
|
||||
new AbortController().signal,
|
||||
),
|
||||
).rejects.toThrow('API key for provider "klingai" is not configured')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,302 @@
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { EnabledMediaModel } from '@shumai/dtos'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { Type, type Static, type TSchema } from 'typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { ulid } from 'ulid'
|
||||
import { resolveS3MediaBuffer } from './download-asset'
|
||||
|
||||
export function createGenerateVideoTool(
|
||||
enabledModels: EnabledMediaModel[],
|
||||
providerKeys: Record<string, { apiKey?: string }>,
|
||||
userId?: string,
|
||||
): AgentTool {
|
||||
const modelIdentifiers = enabledModels.map((m) => `${m.provider}:${m.modelId}`)
|
||||
const modelSchemas =
|
||||
modelIdentifiers.length === 1
|
||||
? Type.Literal(modelIdentifiers[0])
|
||||
: Type.Union(
|
||||
modelIdentifiers.map((id) => Type.Literal(id)) as unknown as [
|
||||
TSchema,
|
||||
TSchema,
|
||||
...TSchema[],
|
||||
],
|
||||
)
|
||||
|
||||
const generateVideoSchema = Type.Object(
|
||||
{
|
||||
mode: Type.Union(
|
||||
[
|
||||
Type.Literal('text_to_video'),
|
||||
Type.Literal('image_to_video'),
|
||||
Type.Literal('first_last_frame'),
|
||||
Type.Literal('reference_to_video'),
|
||||
],
|
||||
{
|
||||
description:
|
||||
'The video generation mode: "text_to_video", "image_to_video", "first_last_frame", or "reference_to_video".',
|
||||
},
|
||||
),
|
||||
model: Type.Union([modelSchemas, Type.Null()], {
|
||||
description: `The video model to use, in "provider:modelId" format. Available enabled models: ${modelIdentifiers.join(', ')}. Pass null to use the first enabled model (${modelIdentifiers[0]}).`,
|
||||
}),
|
||||
outputFileName: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Optional desired filename for the generated video (e.g. "intro.mp4"). Set to null for auto-generated name.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
textToVideoConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
prompt: Type.String({
|
||||
description: 'Text prompt describing the desired video scene, action, and motion.',
|
||||
}),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
imageToVideoConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
image: Type.String({
|
||||
description:
|
||||
'Storage S3 key of the image to animate. Call read_asset with s3KeyOnly: true to obtain the S3 key of an image asset or video frame without loading its content into context.',
|
||||
}),
|
||||
prompt: Type.Union([
|
||||
Type.String({
|
||||
description: 'Optional motion prompt describing how the image should animate.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
firstLastFrameConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
firstFrame: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Storage S3 key of the starting frame image. Call read_asset with s3KeyOnly: true to obtain the S3 key.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
lastFrame: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Storage S3 key of the ending frame image. Call read_asset with s3KeyOnly: true to obtain the S3 key.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
prompt: Type.Union([
|
||||
Type.String({ description: 'Optional prompt describing the transition.' }),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
referenceToVideoConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
references: Type.Array(Type.String(), {
|
||||
description:
|
||||
'List of storage S3 keys for style/motion/character reference images. Call read_asset with s3KeyOnly: true to obtain the S3 keys.',
|
||||
}),
|
||||
prompt: Type.Union([
|
||||
Type.String({ description: 'Prompt describing the video to generate.' }),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
aspectRatio: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'Optional aspect ratio: "16:9", "9:16", "1:1", or "adaptive", or null for default.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
resolution: Type.Union([
|
||||
Type.String({
|
||||
description: 'Optional target video resolution, e.g. "1280x720", or null for default.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
duration: Type.Union([
|
||||
Type.Integer({ description: 'Optional duration in seconds, or null for default.' }),
|
||||
Type.Null(),
|
||||
]),
|
||||
fps: Type.Union([
|
||||
Type.Integer({ description: 'Optional frames per second, or null for default.' }),
|
||||
Type.Null(),
|
||||
]),
|
||||
generateAudio: Type.Union([
|
||||
Type.Boolean({
|
||||
description: 'Whether to generate audio alongside video, or null for default.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
seed: Type.Union([
|
||||
Type.Integer({
|
||||
description:
|
||||
'Optional random seed for reproducible video generation, or null for random.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
)
|
||||
|
||||
type GenerateVideoParams = Static<typeof generateVideoSchema>
|
||||
|
||||
return {
|
||||
name: 'generate_video',
|
||||
label: 'Generate Video',
|
||||
description:
|
||||
'Generate a video using a configured video model. Supports text-to-video, image-to-video (using an S3 image key from read_asset), first/last frame (using S3 image keys from read_asset), and reference-to-video (using S3 image keys from read_asset). The generated file is saved in the local .pi/ directory.',
|
||||
parameters: generateVideoSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const p = params as GenerateVideoParams
|
||||
|
||||
const defaultModel = `${enabledModels[0].provider}:${enabledModels[0].modelId}`
|
||||
const selected = p.model || defaultModel
|
||||
const modelConfig =
|
||||
enabledModels.find((m) => `${m.provider}:${m.modelId}` === selected) ||
|
||||
enabledModels.find((m) => m.modelId === selected)
|
||||
|
||||
if (!modelConfig) {
|
||||
throw new Error(
|
||||
`Video model "${selected}" is not enabled. Available enabled models: ${modelIdentifiers.join(', ')}`,
|
||||
)
|
||||
}
|
||||
|
||||
const customKey = providerKeys[modelConfig.provider]?.apiKey
|
||||
const apiKey = mediaGenerationService.resolveApiKey(modelConfig.provider, customKey)
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
`API key for provider "${modelConfig.provider}" is not configured. Please configure it in settings.`,
|
||||
)
|
||||
}
|
||||
|
||||
let promptText = ''
|
||||
let resolvedImage: Buffer | undefined
|
||||
let resolvedFirstFrame: Buffer | undefined
|
||||
let resolvedLastFrame: Buffer | undefined
|
||||
let resolvedReferences: Buffer[] | undefined
|
||||
|
||||
if (p.mode === 'text_to_video') {
|
||||
if (!p.textToVideoConfig?.prompt) {
|
||||
throw new Error('textToVideoConfig.prompt is required when mode is "text_to_video"')
|
||||
}
|
||||
promptText = p.textToVideoConfig.prompt
|
||||
} else if (p.mode === 'image_to_video') {
|
||||
if (!p.imageToVideoConfig?.image) {
|
||||
throw new Error('imageToVideoConfig.image is required when mode is "image_to_video"')
|
||||
}
|
||||
const { buffer } = await resolveS3MediaBuffer(p.imageToVideoConfig.image, userId || '')
|
||||
resolvedImage = buffer
|
||||
promptText = p.imageToVideoConfig.prompt || ''
|
||||
} else if (p.mode === 'first_last_frame') {
|
||||
if (!p.firstLastFrameConfig?.firstFrame && !p.firstLastFrameConfig?.lastFrame) {
|
||||
throw new Error(
|
||||
'At least one of firstFrame or lastFrame is required in firstLastFrameConfig',
|
||||
)
|
||||
}
|
||||
if (p.firstLastFrameConfig.firstFrame) {
|
||||
const { buffer } = await resolveS3MediaBuffer(
|
||||
p.firstLastFrameConfig.firstFrame,
|
||||
userId || '',
|
||||
)
|
||||
resolvedFirstFrame = buffer
|
||||
}
|
||||
if (p.firstLastFrameConfig.lastFrame) {
|
||||
const { buffer } = await resolveS3MediaBuffer(
|
||||
p.firstLastFrameConfig.lastFrame,
|
||||
userId || '',
|
||||
)
|
||||
resolvedLastFrame = buffer
|
||||
}
|
||||
promptText = p.firstLastFrameConfig.prompt || ''
|
||||
} else if (p.mode === 'reference_to_video') {
|
||||
if (
|
||||
!p.referenceToVideoConfig?.references ||
|
||||
p.referenceToVideoConfig.references.length === 0
|
||||
) {
|
||||
throw new Error('referenceToVideoConfig.references list is required')
|
||||
}
|
||||
resolvedReferences = await Promise.all(
|
||||
p.referenceToVideoConfig.references.map(async (ref: string) => {
|
||||
const { buffer } = await resolveS3MediaBuffer(ref, userId || '')
|
||||
return buffer
|
||||
}),
|
||||
)
|
||||
promptText = p.referenceToVideoConfig.prompt || ''
|
||||
}
|
||||
|
||||
const { buffer, mimeType } = await mediaGenerationService.generateVideo({
|
||||
provider: modelConfig.provider,
|
||||
modelId: modelConfig.modelId,
|
||||
apiKey,
|
||||
mode: p.mode,
|
||||
prompt: promptText,
|
||||
image: resolvedImage,
|
||||
firstFrame: resolvedFirstFrame,
|
||||
lastFrame: resolvedLastFrame,
|
||||
inputReferences: resolvedReferences,
|
||||
aspectRatio: (p.aspectRatio as `${number}:${number}` | 'adaptive') || undefined,
|
||||
resolution: (p.resolution as `${number}x${number}`) || undefined,
|
||||
duration: p.duration !== null && p.duration !== undefined ? p.duration : undefined,
|
||||
fps: p.fps !== null && p.fps !== undefined ? p.fps : undefined,
|
||||
generateAudio: p.generateAudio !== null ? p.generateAudio : undefined,
|
||||
seed: p.seed !== null && p.seed !== undefined ? p.seed : undefined,
|
||||
})
|
||||
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
if (!fs.existsSync(piDir)) {
|
||||
fs.mkdirSync(piDir, { recursive: true })
|
||||
}
|
||||
|
||||
let filename: string
|
||||
if (p.outputFileName) {
|
||||
const safe = sanitizeFilename(p.outputFileName)
|
||||
filename = safe.endsWith('.mp4') ? safe : `${safe}.mp4`
|
||||
} else {
|
||||
filename = `generated_video_${ulid()}.mp4`
|
||||
}
|
||||
|
||||
const targetPath = path.join(piDir, filename)
|
||||
const relativePath = path.join('.pi', filename)
|
||||
fs.writeFileSync(targetPath, buffer)
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Successfully generated video using model "${modelConfig.modelId}" (${modelConfig.provider}) in mode "${p.mode}". Saved to "${relativePath}". Size: ${(buffer.length / (1024 * 1024)).toFixed(2)} MB.`,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
fileName: filename,
|
||||
filePath: relativePath,
|
||||
absolutePath: targetPath,
|
||||
size: buffer.length,
|
||||
mimeType,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.modelId,
|
||||
mode: p.mode,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -48,7 +48,8 @@ describe('createListAssetsTool', () => {
|
||||
after: undefined,
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
assets: [
|
||||
{ id: '01M2FILE2', name: 'file2.txt', type: 'file', size: 200 },
|
||||
{ id: '01M1FILE1', name: 'file1.txt', type: 'file', size: 100 },
|
||||
|
||||
@@ -70,7 +70,7 @@ export function createListAssetsTool(userId: string): AgentTool<typeof listAsset
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(output, null, 2) }],
|
||||
details: output,
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createMoveAssetsTool } from './move-assets'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
Permission: { Edit: 'edit', Read: 'read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/asset/asset', () => ({
|
||||
assetService: {
|
||||
reparentAssets: vi.fn().mockResolvedValue(undefined),
|
||||
getAssetContext: vi.fn().mockResolvedValue({
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
auditLogService: {
|
||||
logAction: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('createMoveAssetsTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('checks permissions on destination and all assets, and calls reparentAssets', async () => {
|
||||
const tool = createMoveAssetsTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetIds: ['asset-1', 'asset-2'],
|
||||
newParentId: 'folder-dest',
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'edit',
|
||||
type: 'asset',
|
||||
id: 'folder-dest',
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'edit',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'edit',
|
||||
type: 'asset',
|
||||
id: 'asset-2',
|
||||
})
|
||||
|
||||
expect(assetService.reparentAssets).toHaveBeenCalledWith({
|
||||
assetIds: ['asset-1', 'asset-2'],
|
||||
newParentId: 'folder-dest',
|
||||
creatorId: 'user-1',
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
success: true,
|
||||
movedAssetIds: ['asset-1', 'asset-2'],
|
||||
newParentId: 'folder-dest',
|
||||
})
|
||||
})
|
||||
|
||||
it('logs asset_reparent for each asset when agentContext is provided', async () => {
|
||||
const tool = createMoveAssetsTool('user-1', { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', {
|
||||
assetIds: ['asset-1', 'asset-2'],
|
||||
newParentId: 'folder-dest',
|
||||
})
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledTimes(2)
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.asset_reparent,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: 'proj-1',
|
||||
itemId: 'asset-1',
|
||||
})
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.asset_reparent,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: 'proj-1',
|
||||
itemId: 'asset-2',
|
||||
})
|
||||
})
|
||||
|
||||
it('fails when permission check on target folder fails', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValueOnce(
|
||||
new Error('Access denied to destination'),
|
||||
)
|
||||
const tool = createMoveAssetsTool('user-1')
|
||||
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetIds: ['asset-1'],
|
||||
newParentId: 'folder-dest',
|
||||
}),
|
||||
).rejects.toThrow('Access denied to destination')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,84 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { type User } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
const moveAssetsSchema = Type.Object({
|
||||
assetIds: Type.Array(Type.String(), {
|
||||
description:
|
||||
'The IDs of the assets to move. All assets must belong to the same parent folder. At least one asset ID is required.',
|
||||
minItems: 1,
|
||||
}),
|
||||
newParentId: Type.String({
|
||||
description:
|
||||
'The destination parent folder ID to move the assets into. This parameter is required.',
|
||||
}),
|
||||
})
|
||||
|
||||
export function createMoveAssetsTool(
|
||||
userId: string,
|
||||
agentContext?: { teamId?: string; agentId?: string },
|
||||
): AgentTool<typeof moveAssetsSchema> {
|
||||
return {
|
||||
name: 'move_assets',
|
||||
label: 'Move Assets',
|
||||
description:
|
||||
'Move one or more assets (files or folders) to a specified parent folder. Note: All assets to be moved must belong to the same current parent folder.',
|
||||
parameters: moveAssetsSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
// Check Edit permission on destination folder
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Asset,
|
||||
id: params.newParentId,
|
||||
})
|
||||
|
||||
// Check Edit permission on all source assets
|
||||
for (const assetId of params.assetIds) {
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
}
|
||||
|
||||
await assetService.reparentAssets({
|
||||
assetIds: params.assetIds,
|
||||
newParentId: params.newParentId,
|
||||
creatorId: userId,
|
||||
})
|
||||
|
||||
if (agentContext?.teamId) {
|
||||
const ctx = await assetService
|
||||
.getAssetContext(params.newParentId)
|
||||
.catch(() => ({ teamId: agentContext.teamId!, projectId: undefined }))
|
||||
for (const assetId of params.assetIds) {
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.asset_reparent,
|
||||
teamId: agentContext.teamId || ctx.teamId,
|
||||
userId,
|
||||
agentId: agentContext.agentId,
|
||||
projectId: ctx.projectId,
|
||||
itemId: assetId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
movedAssetIds: params.assetIds,
|
||||
newParentId: params.newParentId,
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,644 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createReadAssetTool } from './read-asset'
|
||||
import {
|
||||
prisma,
|
||||
WorkflowTaskType,
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import * as annotationResolver from './annotation-resolver'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
findFirst: vi.fn(),
|
||||
},
|
||||
workflowTask: {
|
||||
create: vi.fn(),
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
getObject: vi.fn(),
|
||||
presign: vi.fn().mockResolvedValue('https://mock-presigned-url.com'),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/workflow-core', () => ({
|
||||
workflowService: {
|
||||
executeWait: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('readAssetTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
|
||||
annotations: null,
|
||||
timestamp: null,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw authorization error if user ID is empty', async () => {
|
||||
const tool = createReadAssetTool('')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'asset-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('User ID is required for authorization.')
|
||||
})
|
||||
|
||||
it('should throw error if asset not found', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'asset-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('Asset with ID asset-1 not found.')
|
||||
})
|
||||
|
||||
describe('Image Assets', () => {
|
||||
it('should read image and return ImageContent and S3 key', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'img-1',
|
||||
name: 'photo.png',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'raw/photo.png' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/photo.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-image-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'img-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'img-1',
|
||||
})
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/photo.webp')
|
||||
|
||||
const textItem = result.content[0] as { type: 'text'; text: string }
|
||||
const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
|
||||
|
||||
expect(textItem.text).toContain('proxy/photo.webp')
|
||||
expect(imageItem.type).toBe('image')
|
||||
expect(imageItem.data).toBe(Buffer.from('fake-image-bytes').toString('base64'))
|
||||
expect(result.details.key).toBe('proxy/photo.webp')
|
||||
})
|
||||
|
||||
it('should trigger image annotation transcode workflow when annotationId is provided', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'img-1',
|
||||
name: 'photo.png',
|
||||
mediaType: 'image/png',
|
||||
projectId: 'project-1',
|
||||
storageKey: { key: 'raw/photo.png' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/photo.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
|
||||
annotations: [
|
||||
{
|
||||
type: 'arrow',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[10, 10],
|
||||
],
|
||||
},
|
||||
],
|
||||
timestamp: null,
|
||||
})
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-anno',
|
||||
} as unknown as WorkflowTask)
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-anno',
|
||||
output: { key: 'annotated/photo.webp' },
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('annotated-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'img-1',
|
||||
annotationId: 'comment-1',
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'img-1',
|
||||
projectId: 'project-1',
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
imageAnnotation: {
|
||||
annotations: [
|
||||
{
|
||||
type: 'arrow',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[10, 10],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'annotated/photo.webp')
|
||||
expect(result.details.key).toBe('annotated/photo.webp')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Video Assets', () => {
|
||||
it('should throw error if videoConfig is missing for video assets', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'vid-1',
|
||||
name: 'video.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
media: { proxyType: 'video' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'vid-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('videoConfig with start, end, and count is required')
|
||||
})
|
||||
|
||||
it('should throw error on invalid video time range', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'vid-1',
|
||||
name: 'video.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
media: { proxyType: 'video' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'vid-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: { start: 10, end: 5, count: 1 },
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
'Invalid video time range: start (10) must be less than or equal to end (5).',
|
||||
)
|
||||
})
|
||||
|
||||
it('should extract frames and return ImageContent array and keys', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'vid-1',
|
||||
name: 'video.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'video' },
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-shot',
|
||||
} as unknown as WorkflowTask)
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-shot',
|
||||
output: {
|
||||
screenshots: [
|
||||
{ key: 'screenshots/shot1.webp', timestamp: 2.0 },
|
||||
{ key: 'screenshots/shot2.webp', timestamp: 4.0 },
|
||||
],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockImplementation(async (_bucket, key) => ({
|
||||
buffer: Buffer.from(`bytes-for-${key}`),
|
||||
contentType: 'image/webp',
|
||||
}))
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'vid-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: { start: 2.0, end: 4.0, count: 2 },
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'vid-1',
|
||||
projectId: 'project-1',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
screenshot: {
|
||||
start: 2.0,
|
||||
end: 4.0,
|
||||
count: 2,
|
||||
commentTimestamp: null,
|
||||
annotations: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.content.length).toBe(3) // 1 text + 2 images
|
||||
expect(result.details.keys).toEqual(['screenshots/shot1.webp', 'screenshots/shot2.webp'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Document Assets', () => {
|
||||
it('should throw error if docConfig is missing for document assets', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
name: 'doc.pdf',
|
||||
mediaType: 'application/pdf',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'doc-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('docConfig with mode ("pages" or "text") is required')
|
||||
})
|
||||
|
||||
it('should render PDF pages in mode: "pages"', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
name: 'doc.pdf',
|
||||
mediaType: 'application/pdf',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-pdf',
|
||||
} as unknown as WorkflowTask)
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-pdf',
|
||||
output: {
|
||||
pages: [
|
||||
{ key: 'pdf_pages/page1.webp', page: 1 },
|
||||
{ key: 'pdf_pages/page2.webp', page: 2 },
|
||||
],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockImplementation(async (_bucket, key) => ({
|
||||
buffer: Buffer.from(`bytes-for-${key}`),
|
||||
contentType: 'image/webp',
|
||||
}))
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'doc-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'pages', startPage: 1, endPage: 2 },
|
||||
})
|
||||
|
||||
expect(result.content.length).toBe(3) // 1 text + 2 images
|
||||
expect(result.details.keys).toEqual(['pdf_pages/page1.webp', 'pdf_pages/page2.webp'])
|
||||
})
|
||||
|
||||
it('should throw error in mode: "pages" when page range exceeds 20 pages', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
name: 'doc.pdf',
|
||||
mediaType: 'application/pdf',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'doc-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'pages', startPage: 1, endPage: 25 },
|
||||
}),
|
||||
).rejects.toThrow('exceeds the maximum limit of 20 pages')
|
||||
})
|
||||
|
||||
it('should throw error in mode: "text" when called on binary PDF', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
name: 'manual.pdf',
|
||||
mediaType: 'application/pdf',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'doc-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'text', startPage: null, endPage: null },
|
||||
}),
|
||||
).rejects.toThrow('is a binary PDF/document and cannot be read as raw text')
|
||||
})
|
||||
|
||||
it('should throw error in mode: "text" when annotationId is provided', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
name: 'notes.md',
|
||||
mediaType: 'text/markdown',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'doc-1',
|
||||
annotationId: 'comment-123',
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'text', startPage: null, endPage: null },
|
||||
}),
|
||||
).rejects.toThrow('annotationId cannot be used with docConfig mode "text"')
|
||||
})
|
||||
|
||||
it('should throw error in mode: "text" when startPage or endPage is non-null', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-1',
|
||||
name: 'notes.md',
|
||||
mediaType: 'text/markdown',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-1', {
|
||||
assetId: 'doc-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'text', startPage: 1, endPage: 2 },
|
||||
}),
|
||||
).rejects.toThrow('startPage and endPage must be null when docConfig mode is "text"')
|
||||
})
|
||||
|
||||
it('should read raw text from S3 in mode: "text" for markdown files', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-md',
|
||||
name: 'README.md',
|
||||
mediaType: 'text/markdown',
|
||||
storageKey: { key: 'raw/README.md' },
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('# Hello Shumai\nThis is markdown text.'),
|
||||
contentType: 'text/markdown',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'doc-md',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'text', startPage: null, endPage: null },
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'raw/README.md')
|
||||
expect(result.content[0].type).toBe('text')
|
||||
expect((result.content[0] as { type: 'text'; text: string }).text).toContain(
|
||||
'# Hello Shumai\nThis is markdown text.',
|
||||
)
|
||||
expect(result.details.key).toBe('raw/README.md')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Version Stack Resolution', () => {
|
||||
it('should resolve version_stack to latest version asset', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'stack-1',
|
||||
name: '',
|
||||
type: 'version_stack',
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.asset.findFirst).mockResolvedValue({
|
||||
id: 'file-v2',
|
||||
name: 'diagram_v2.png',
|
||||
type: 'file',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'raw/diagram_v2.png' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/diagram_v2.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('v2-image-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'stack-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(prisma.asset.findFirst).toHaveBeenCalledWith({
|
||||
where: { parentId: 'stack-1', isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
expect(result.details.name).toBe('diagram_v2.png')
|
||||
expect(result.details.key).toBe('proxy/diagram_v2.webp')
|
||||
})
|
||||
})
|
||||
|
||||
describe('s3KeyOnly mode', () => {
|
||||
it('returns only S3 key and MIME in text for image assets without fetching S3 buffer or attaching base64 image', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'img-1',
|
||||
name: 'photo.png',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'raw/photo.png' },
|
||||
media: {
|
||||
proxyType: 'image',
|
||||
imageTranscodes: [{ key: 'proxy/photo.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'img-1',
|
||||
annotationId: null,
|
||||
s3KeyOnly: true,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).not.toHaveBeenCalled()
|
||||
expect(result.content.length).toBe(1)
|
||||
expect(result.content[0].type).toBe('text')
|
||||
expect((result.content[0] as { type: 'text'; text: string }).text).toBe(
|
||||
'Image asset "photo.png" (ID: img-1, MIME: image/webp, S3 Key: "proxy/photo.webp")',
|
||||
)
|
||||
expect(result.details.key).toBe('proxy/photo.webp')
|
||||
})
|
||||
|
||||
it('returns only frame S3 keys and MIME in text for video assets without fetching S3 buffers or attaching base64 images', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'vid-1',
|
||||
name: 'clip.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'video' },
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-shot',
|
||||
} as unknown as WorkflowTask)
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-shot',
|
||||
output: {
|
||||
screenshots: [{ key: 'screenshots/shot1.webp', timestamp: 1.5 }],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'vid-1',
|
||||
annotationId: null,
|
||||
s3KeyOnly: true,
|
||||
imageConfig: null,
|
||||
videoConfig: { start: 1.5, end: 1.5, count: 1 },
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).not.toHaveBeenCalled()
|
||||
expect(result.content.length).toBe(1)
|
||||
expect(result.content[0].type).toBe('text')
|
||||
const text = (result.content[0] as { type: 'text'; text: string }).text
|
||||
expect(text).toContain('- Frame at 1.5s (S3 Key: "screenshots/shot1.webp", MIME: image/webp)')
|
||||
expect(result.details.keys).toEqual(['screenshots/shot1.webp'])
|
||||
})
|
||||
|
||||
it('returns only document S3 key and MIME in text for text mode without fetching S3 buffer', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'doc-md',
|
||||
name: 'notes.md',
|
||||
mediaType: 'text/markdown',
|
||||
storageKey: { key: 'raw/notes.md' },
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', {
|
||||
assetId: 'doc-md',
|
||||
annotationId: null,
|
||||
s3KeyOnly: true,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: { mode: 'text', startPage: null, endPage: null },
|
||||
})
|
||||
|
||||
expect(s3Service.getObject).not.toHaveBeenCalled()
|
||||
expect(result.content.length).toBe(1)
|
||||
expect(result.content[0].type).toBe('text')
|
||||
expect((result.content[0] as { type: 'text'; text: string }).text).toBe(
|
||||
'Document "notes.md" (ID: doc-md, MIME: text/markdown, S3 Key: "raw/notes.md")',
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,638 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent, type TextContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { resolveAnnotationsById } from './annotation-resolver'
|
||||
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
|
||||
|
||||
export const readAssetSchema = Type.Object(
|
||||
{
|
||||
assetId: Type.String({
|
||||
description: 'The asset ID of the workspace asset to inspect. Required.',
|
||||
}),
|
||||
annotationId: Type.Union([
|
||||
Type.String({
|
||||
description:
|
||||
'The ID of the comment or message entry from <annotation id="..." /> whose visual markup should be overlaid, or null.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
s3KeyOnly: Type.Optional(
|
||||
Type.Boolean({
|
||||
description:
|
||||
'When true, returns only the storage S3 key(s) and MIME types of the asset/frames without attaching base64 image or text content to the agent context. Use this to obtain S3 keys for generate_image or generate_video references without consuming context tokens.',
|
||||
default: false,
|
||||
}),
|
||||
),
|
||||
imageConfig: Type.Union([
|
||||
Type.Object(
|
||||
{},
|
||||
{
|
||||
additionalProperties: false,
|
||||
description: 'Optional configuration for image assets, or null.',
|
||||
},
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
videoConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
start: Type.Number({ description: 'Start time in seconds.' }),
|
||||
end: Type.Number({ description: 'End time in seconds.' }),
|
||||
count: Type.Number({ description: 'Number of frames to extract (e.g. 1 to 10).' }),
|
||||
},
|
||||
{
|
||||
additionalProperties: false,
|
||||
description:
|
||||
'Configuration for video assets. Required when inspecting videos, or null for other asset types.',
|
||||
},
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
docConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
mode: Type.Union([
|
||||
Type.Literal('pages', {
|
||||
description:
|
||||
'Render specific page range as images for visual layout, charts, or visual annotations (required for binary PDFs).',
|
||||
}),
|
||||
Type.Literal('text', {
|
||||
description:
|
||||
'Read raw text content directly (ideal for .md, .txt, .csv, code files).',
|
||||
}),
|
||||
]),
|
||||
startPage: Type.Union([
|
||||
Type.Number({
|
||||
description:
|
||||
'Start page number (1-based index, required when mode="pages"), or null when mode="text".',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
endPage: Type.Union([
|
||||
Type.Number({
|
||||
description:
|
||||
'End page number (1-based index, max 20 pages per call, required when mode="pages"), or null when mode="text".',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{
|
||||
additionalProperties: false,
|
||||
description:
|
||||
'Configuration for document/PDF/text assets, or null if inspecting another asset type.',
|
||||
},
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
)
|
||||
|
||||
export function isPlainTextAsset(mediaType: string, filename: string): boolean {
|
||||
if (
|
||||
mediaType.startsWith('text/') ||
|
||||
mediaType === 'application/json' ||
|
||||
mediaType === 'application/javascript' ||
|
||||
mediaType === 'application/typescript' ||
|
||||
mediaType === 'application/xml' ||
|
||||
mediaType === 'application/x-yaml'
|
||||
) {
|
||||
return true
|
||||
}
|
||||
const textExtensions = [
|
||||
'.txt',
|
||||
'.md',
|
||||
'.markdown',
|
||||
'.json',
|
||||
'.csv',
|
||||
'.tsv',
|
||||
'.js',
|
||||
'.ts',
|
||||
'.tsx',
|
||||
'.jsx',
|
||||
'.html',
|
||||
'.htm',
|
||||
'.css',
|
||||
'.scss',
|
||||
'.yaml',
|
||||
'.yml',
|
||||
'.xml',
|
||||
'.py',
|
||||
'.rb',
|
||||
'.go',
|
||||
'.rs',
|
||||
'.c',
|
||||
'.cpp',
|
||||
'.h',
|
||||
'.hpp',
|
||||
'.java',
|
||||
'.sh',
|
||||
'.bash',
|
||||
'.zsh',
|
||||
'.sql',
|
||||
]
|
||||
return textExtensions.some((ext) => filename.endsWith(ext))
|
||||
}
|
||||
|
||||
const MAX_TEXT_BYTES = 50 * 1024
|
||||
const MAX_TEXT_LINES = 1000
|
||||
|
||||
export function createReadAssetTool(userId: string): AgentTool<typeof readAssetSchema> {
|
||||
return {
|
||||
name: 'read_asset',
|
||||
label: 'Read Asset',
|
||||
description:
|
||||
'Inspects and reads the content of an asset in the workspace. ' +
|
||||
'Supports visual analysis for images, video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
|
||||
'Pass assetId, and provide the specific config (imageConfig, videoConfig, or docConfig) while setting unused configs to null. ' +
|
||||
'Optionally provide annotationId to view visual markups drawn on the asset.',
|
||||
parameters: readAssetSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const { assetId, annotationId, s3KeyOnly, videoConfig, docConfig } = params
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
let asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
if (asset.type === 'version_stack') {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (latestVersion) {
|
||||
asset = latestVersion
|
||||
}
|
||||
}
|
||||
|
||||
const mediaInfo = asset.media as unknown as PrismaJson.MediaInfo | null
|
||||
const proxyType = mediaInfo?.proxyType
|
||||
const mediaType = asset.mediaType?.toLowerCase() || ''
|
||||
const filename = asset.name?.toLowerCase() || ''
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Branch 1: Image Assets
|
||||
// ----------------------------------------------------------------------
|
||||
if (proxyType === 'image' || mediaType.startsWith('image/') || filename.endsWith('.psd')) {
|
||||
let mediaKey = asset.storageKey?.key
|
||||
if (mediaInfo?.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
|
||||
}
|
||||
|
||||
if (!mediaKey) {
|
||||
throw new Error('No media content found for this asset.')
|
||||
}
|
||||
|
||||
let keyToUse = mediaKey
|
||||
let mimeType = getFileMimeType(null, keyToUse, asset.mediaType || 'image/png')
|
||||
|
||||
if (annotationId) {
|
||||
const { annotations } = await resolveAnnotationsById(asset.id, annotationId)
|
||||
if (annotations && annotations.length > 0) {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
imageAnnotation: {
|
||||
annotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode image annotation task output
|
||||
const output = completedTask.output as any
|
||||
if (output?.key) {
|
||||
keyToUse = output.key
|
||||
mimeType = 'image/webp'
|
||||
} else {
|
||||
throw new Error('Image annotation transcode workflow failed to output S3 key.')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const downloadUrl = await s3Service.presign(
|
||||
bucket,
|
||||
keyToUse,
|
||||
'GET',
|
||||
true,
|
||||
asset.name || undefined,
|
||||
)
|
||||
|
||||
const content: Array<TextContent | ImageContent> = []
|
||||
|
||||
if (s3KeyOnly) {
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: `Image asset "${asset.name}" (ID: ${asset.id}, MIME: ${mimeType}, S3 Key: "${keyToUse}")`,
|
||||
})
|
||||
} else {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
|
||||
const actualMimeType =
|
||||
contentType && contentType !== 'application/octet-stream' ? contentType : mimeType
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: `Image asset "${asset.name}" (ID: ${asset.id}, MIME: ${actualMimeType}, S3 Key: "${keyToUse}")`,
|
||||
})
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: actualMimeType,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
key: keyToUse,
|
||||
downloadUrl,
|
||||
sourceKeys: [keyToUse],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Branch 2: Video Assets
|
||||
// ----------------------------------------------------------------------
|
||||
if (proxyType === 'video' || mediaType.startsWith('video/')) {
|
||||
if (!videoConfig) {
|
||||
throw new Error(
|
||||
'videoConfig with start, end, and count is required when inspecting video assets.',
|
||||
)
|
||||
}
|
||||
|
||||
const { start, end, count } = videoConfig
|
||||
|
||||
if (start < 0 || end < 0) {
|
||||
throw new Error('Invalid video time range: start and end must be non-negative numbers.')
|
||||
}
|
||||
|
||||
if (start > end) {
|
||||
throw new Error(
|
||||
`Invalid video time range: start (${start}) must be less than or equal to end (${end}).`,
|
||||
)
|
||||
}
|
||||
|
||||
if (count < 1) {
|
||||
throw new Error(`Invalid frame count: count (${count}) must be at least 1.`)
|
||||
}
|
||||
|
||||
const { annotations, timestamp: commentTimestamp } = await resolveAnnotationsById(
|
||||
asset.id,
|
||||
annotationId,
|
||||
)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
screenshot: {
|
||||
start,
|
||||
end,
|
||||
count,
|
||||
commentTimestamp,
|
||||
annotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode screenshot output
|
||||
const output = completedTask.output as any
|
||||
const screenshots = output?.screenshots as
|
||||
| Array<{ key: string; timestamp: number }>
|
||||
| undefined
|
||||
|
||||
if (!screenshots || screenshots.length === 0) {
|
||||
throw new Error('No screenshots were generated.')
|
||||
}
|
||||
|
||||
const content: Array<TextContent | ImageContent> = []
|
||||
const sourceKeys: string[] = []
|
||||
const results: Array<{ key: string; timestamp: number; downloadUrl: string }> = []
|
||||
const textLines: string[] = [
|
||||
`Extracted ${screenshots.length} frame(s) from "${asset.name}" (${start}s - ${end}s):`,
|
||||
]
|
||||
|
||||
for (const shot of screenshots) {
|
||||
const downloadUrl = await s3Service.presign(
|
||||
bucket,
|
||||
shot.key,
|
||||
'GET',
|
||||
true,
|
||||
asset.name || undefined,
|
||||
)
|
||||
|
||||
sourceKeys.push(shot.key)
|
||||
results.push({
|
||||
key: shot.key,
|
||||
timestamp: shot.timestamp,
|
||||
downloadUrl,
|
||||
})
|
||||
|
||||
if (s3KeyOnly) {
|
||||
const mimeType = getFileMimeType(null, shot.key, 'image/webp')
|
||||
textLines.push(
|
||||
`- Frame at ${shot.timestamp}s (S3 Key: "${shot.key}", MIME: ${mimeType})`,
|
||||
)
|
||||
} else {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, shot.key)
|
||||
const mimeType =
|
||||
contentType && contentType !== 'application/octet-stream'
|
||||
? contentType
|
||||
: getFileMimeType(buffer, shot.key, 'image/webp')
|
||||
textLines.push(
|
||||
`- Frame at ${shot.timestamp}s (S3 Key: "${shot.key}", MIME: ${mimeType})`,
|
||||
)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
content.unshift({
|
||||
type: 'text',
|
||||
text: textLines.join('\n'),
|
||||
})
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
keys: sourceKeys,
|
||||
results,
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Branch 3: Documents & PDF Assets
|
||||
// ----------------------------------------------------------------------
|
||||
if (
|
||||
proxyType === 'pdf' ||
|
||||
mediaType === 'application/pdf' ||
|
||||
isPlainTextAsset(mediaType, filename)
|
||||
) {
|
||||
if (!docConfig) {
|
||||
throw new Error(
|
||||
'docConfig with mode ("pages" or "text") is required when inspecting document/PDF assets.',
|
||||
)
|
||||
}
|
||||
|
||||
if (docConfig.mode === 'text') {
|
||||
if (annotationId !== null) {
|
||||
throw new Error(
|
||||
'annotationId cannot be used with docConfig mode "text" because visual markups can only be rendered on visual pages. Use docConfig: { mode: "pages", startPage: ..., endPage: ... } instead.',
|
||||
)
|
||||
}
|
||||
|
||||
if (docConfig.startPage !== null || docConfig.endPage !== null) {
|
||||
throw new Error(
|
||||
'startPage and endPage must be null when docConfig mode is "text". To inspect specific page numbers, use mode: "pages".',
|
||||
)
|
||||
}
|
||||
|
||||
if (!isPlainTextAsset(mediaType, filename)) {
|
||||
throw new Error(
|
||||
`Asset "${asset.name}" is a binary PDF/document and cannot be read as raw text. Please call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ... } to view its visual pages.`,
|
||||
)
|
||||
}
|
||||
|
||||
const mediaKey = asset.storageKey?.key
|
||||
if (!mediaKey) {
|
||||
throw new Error(`No media content found for asset ${asset.id}.`)
|
||||
}
|
||||
|
||||
const docMimeType =
|
||||
asset.mediaType || getFileMimeType(null, asset.name || mediaKey, 'text/plain')
|
||||
const downloadUrl = await s3Service.presign(
|
||||
bucket,
|
||||
mediaKey,
|
||||
'GET',
|
||||
true,
|
||||
asset.name || undefined,
|
||||
)
|
||||
|
||||
if (s3KeyOnly) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Document "${asset.name}" (ID: ${asset.id}, MIME: ${docMimeType}, S3 Key: "${mediaKey}")`,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
key: mediaKey,
|
||||
downloadUrl,
|
||||
sourceKeys: [mediaKey],
|
||||
size: 0,
|
||||
isTruncated: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const { buffer } = await s3Service.getObject(bucket, mediaKey)
|
||||
const rawText = buffer.toString('utf-8')
|
||||
let text = rawText
|
||||
let isTruncated = false
|
||||
|
||||
const lines = text.split('\n')
|
||||
if (lines.length > MAX_TEXT_LINES) {
|
||||
text = lines.slice(0, MAX_TEXT_LINES).join('\n')
|
||||
isTruncated = true
|
||||
}
|
||||
|
||||
if (Buffer.byteLength(text, 'utf-8') > MAX_TEXT_BYTES) {
|
||||
text = Buffer.from(text, 'utf-8').subarray(0, MAX_TEXT_BYTES).toString('utf-8')
|
||||
isTruncated = true
|
||||
}
|
||||
|
||||
if (isTruncated) {
|
||||
text += `\n\n[Content truncated to ${MAX_TEXT_LINES} lines / ${MAX_TEXT_BYTES / 1024}KB limit. Use download_asset to process the full file.]`
|
||||
}
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Document "${asset.name}" (ID: ${asset.id}, MIME: ${docMimeType}, S3 Key: "${mediaKey}"):\n\n${text}`,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
key: mediaKey,
|
||||
downloadUrl,
|
||||
sourceKeys: [mediaKey],
|
||||
size: buffer.length,
|
||||
isTruncated,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// docConfig.mode === 'pages'
|
||||
if (docConfig.startPage === null || docConfig.endPage === null) {
|
||||
throw new Error('startPage and endPage are required when docConfig mode is "pages".')
|
||||
}
|
||||
|
||||
const start = docConfig.startPage
|
||||
const end = docConfig.endPage
|
||||
|
||||
if (start < 1) {
|
||||
throw new Error(`Invalid page range: start page (${start}) must be at least 1.`)
|
||||
}
|
||||
|
||||
if (start > end) {
|
||||
throw new Error(
|
||||
`Invalid page range: start page (${start}) must be less than or equal to end page (${end}).`,
|
||||
)
|
||||
}
|
||||
|
||||
const MAX_PAGE_RANGE = 20
|
||||
const requestedPages = end - start + 1
|
||||
if (requestedPages > MAX_PAGE_RANGE) {
|
||||
throw new Error(
|
||||
`Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
|
||||
)
|
||||
}
|
||||
|
||||
const { annotations, timestamp: commentTimestamp } = await resolveAnnotationsById(
|
||||
asset.id,
|
||||
annotationId,
|
||||
)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_pdf_pages,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
pdfPages: {
|
||||
start,
|
||||
end,
|
||||
commentTimestamp,
|
||||
annotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode pdf pages output
|
||||
const output = completedTask.output as any
|
||||
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
|
||||
|
||||
if (!pages || pages.length === 0) {
|
||||
throw new Error('No PDF page images were generated.')
|
||||
}
|
||||
|
||||
const content: Array<TextContent | ImageContent> = []
|
||||
const sourceKeys: string[] = []
|
||||
const results: Array<{ key: string; page: number; downloadUrl: string }> = []
|
||||
const textLines: string[] = [
|
||||
`Rendered ${pages.length} page(s) of "${asset.name}" (Pages ${start} to ${end}):`,
|
||||
]
|
||||
|
||||
for (const pageItem of pages) {
|
||||
const downloadUrl = await s3Service.presign(
|
||||
bucket,
|
||||
pageItem.key,
|
||||
'GET',
|
||||
true,
|
||||
asset.name || undefined,
|
||||
)
|
||||
|
||||
sourceKeys.push(pageItem.key)
|
||||
results.push({
|
||||
key: pageItem.key,
|
||||
page: pageItem.page,
|
||||
downloadUrl,
|
||||
})
|
||||
|
||||
if (s3KeyOnly) {
|
||||
const mimeType = getFileMimeType(null, pageItem.key, 'image/webp')
|
||||
textLines.push(`- Page ${pageItem.page} (S3 Key: "${pageItem.key}", MIME: ${mimeType})`)
|
||||
} else {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
|
||||
const mimeType =
|
||||
contentType && contentType !== 'application/octet-stream'
|
||||
? contentType
|
||||
: getFileMimeType(buffer, pageItem.key, 'image/webp')
|
||||
textLines.push(`- Page ${pageItem.page} (S3 Key: "${pageItem.key}", MIME: ${mimeType})`)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
content.unshift({
|
||||
type: 'text',
|
||||
text: textLines.join('\n'),
|
||||
})
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
keys: sourceKeys,
|
||||
results,
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unsupported asset type for read_asset. Asset ${asset.id} has media type "${mediaType || 'unknown'}". Use download_asset to download this file to disk.`,
|
||||
)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createReadPdfPagesTool } from './read-pdf-pages'
|
||||
import {
|
||||
prisma,
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type AssetComment,
|
||||
type User,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
assetComment: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
workflowTask: {
|
||||
create: vi.fn(),
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
getObject: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/workflow-core', () => ({
|
||||
workflowService: {
|
||||
executeWait: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('readPdfPagesTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
})
|
||||
|
||||
it('should trigger pdfPages transcode workflow and return image outputs', async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
|
||||
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'pdf' },
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
second: 2,
|
||||
annotation: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
} as unknown as AssetComment)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
pages: [
|
||||
{ key: 'pdf_pages/page_1.webp', page: 1 },
|
||||
{ key: 'pdf_pages/page_2.webp', page: 2 },
|
||||
],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockImplementation(async (_bucket: string, key: string) => {
|
||||
return {
|
||||
buffer: Buffer.from(`fake-bytes-for-${key}`),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string }
|
||||
})
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 2 })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
type: 'transcode_pdf_pages',
|
||||
status: 'pending',
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
pdfPages: {
|
||||
start: 1,
|
||||
end: 2,
|
||||
commentTimestamp: 2,
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.content.length).toBe(2)
|
||||
expect(result.content[0].type).toBe('image')
|
||||
expect((result.content[0] as unknown as { data: string }).data).toBe(
|
||||
Buffer.from('fake-bytes-for-pdf_pages/page_1.webp').toString('base64'),
|
||||
)
|
||||
expect(result.details.sourceKeys).toEqual(['pdf_pages/page_1.webp', 'pdf_pages/page_2.webp'])
|
||||
})
|
||||
|
||||
it('should throw error if start page is less than 1', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })).rejects.toThrow(
|
||||
'Invalid page range: start page (0) must be at least 1.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if start page is greater than end page', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })).rejects.toThrow(
|
||||
'Invalid page range: start page (5) must be less than or equal to end page (2).',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if page range exceeds maximum limit of 20', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })).rejects.toThrow(
|
||||
'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if asset not found', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
|
||||
'Asset with ID asset-1 not found.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if proxyType is not pdf', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
media: { proxyType: 'image' },
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
|
||||
'Asset asset-1 is not a PDF or document.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if proxyType is null/undefined (unprocessed)', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
media: null,
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
|
||||
'Asset asset-1 is not a PDF or document.',
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -1,152 +0,0 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
const readPdfPagesSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'Asset ID of the PDF document. This parameter is required.',
|
||||
}),
|
||||
start: Type.Number({ description: 'Start page number (1-based index)' }),
|
||||
end: Type.Number({ description: 'End page number (1-based index)' }),
|
||||
})
|
||||
|
||||
export function createReadPdfPagesTool(
|
||||
userId: string,
|
||||
userCommentId?: string | null,
|
||||
): AgentTool<typeof readPdfPagesSchema> {
|
||||
return {
|
||||
name: 'read_pdf_pages',
|
||||
label: 'Read PDF Pages',
|
||||
description:
|
||||
'Converts specific page range of a PDF document into 1080p WebP images for visual analysis. Maximum 20 pages allowed per call.',
|
||||
parameters: readPdfPagesSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
if (params.start < 1) {
|
||||
throw new Error(`Invalid page range: start page (${params.start}) must be at least 1.`)
|
||||
}
|
||||
|
||||
if (params.start > params.end) {
|
||||
throw new Error(
|
||||
`Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
|
||||
)
|
||||
}
|
||||
|
||||
const MAX_PAGE_RANGE = 20
|
||||
const requestedPages = params.end - params.start + 1
|
||||
if (requestedPages > MAX_PAGE_RANGE) {
|
||||
throw new Error(
|
||||
`Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
|
||||
)
|
||||
}
|
||||
|
||||
const targetAssetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: targetAssetId,
|
||||
})
|
||||
|
||||
let asset = await prisma.asset.findUnique({
|
||||
where: { id: targetAssetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${targetAssetId} not found.`)
|
||||
}
|
||||
|
||||
if (asset.type === 'version_stack') {
|
||||
const latestVersion = await prisma.asset.findFirst({
|
||||
where: { parentId: asset.id, isDeleted: false },
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
})
|
||||
if (latestVersion) {
|
||||
asset = latestVersion
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (proxyType !== 'pdf') {
|
||||
throw new Error(`Asset ${targetAssetId} is not a PDF or document.`)
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for comment page and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === targetAssetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger transcode workflow to render PDF pages
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: targetAssetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_pdf_pages,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
pdfPages: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
|
||||
|
||||
if (!pages || pages.length === 0) {
|
||||
throw new Error('No PDF page images were generated.')
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const pageItem of pages) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
|
||||
const mimeType =
|
||||
contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp'
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType,
|
||||
})
|
||||
sourceKeys.push(pageItem.key)
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,8 @@ describe('createReadThreadTool', () => {
|
||||
id: 'root-1',
|
||||
message: 'Initial top-level question',
|
||||
creator: { name: 'Alice' },
|
||||
second: 12.5,
|
||||
annotation: [{ type: 'box' }],
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
vi.mocked(prisma.assetComment.findMany).mockResolvedValue([])
|
||||
@@ -59,7 +61,9 @@ describe('createReadThreadTool', () => {
|
||||
const result = await tool.execute('call-2', { threadId: 'root-1' })
|
||||
const textContent = (result.content[0] as { text: string }).text
|
||||
|
||||
expect(textContent).toContain('Thread Root [Alice] (root-1): Initial top-level question')
|
||||
expect(textContent).toContain(
|
||||
'Thread Root [Alice] (id: root-1, time: 12.5s, has_markup: true): Initial top-level question',
|
||||
)
|
||||
expect(textContent).toContain('(No replies in this thread yet)')
|
||||
})
|
||||
|
||||
@@ -75,6 +79,8 @@ describe('createReadThreadTool', () => {
|
||||
id: 'reply-1',
|
||||
message: 'First reply to thread',
|
||||
creator: { name: 'Bob' },
|
||||
second: 15.0,
|
||||
annotation: [{ type: 'line' }],
|
||||
},
|
||||
{
|
||||
id: 'reply-2',
|
||||
@@ -88,8 +94,45 @@ describe('createReadThreadTool', () => {
|
||||
const result = await tool.execute('call-3', { threadId: 'root-1' })
|
||||
const textContent = (result.content[0] as { text: string }).text
|
||||
|
||||
expect(textContent).toContain('Thread Root [Alice] (root-1): Initial top-level question')
|
||||
expect(textContent).toContain('- [Bob]: First reply to thread')
|
||||
expect(textContent).toContain('- [Charlie]: Second reply to thread')
|
||||
expect(textContent).toContain(
|
||||
'Thread Root [Alice] (id: root-1, has_markup: false): Initial top-level question',
|
||||
)
|
||||
expect(textContent).toContain(
|
||||
'- [Bob] (id: reply-1, time: 15s, has_markup: true): First reply to thread',
|
||||
)
|
||||
expect(textContent).toContain(
|
||||
'- [Charlie] (id: reply-2, has_markup: false): Second reply to thread',
|
||||
)
|
||||
})
|
||||
|
||||
it('should label agent replies as [Ai Agent] when creator is null but sessionId exists', async () => {
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'root-1',
|
||||
message: 'Question to agent',
|
||||
creator: { name: 'Alice' },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
vi.mocked(prisma.assetComment.findMany).mockResolvedValue([
|
||||
{
|
||||
id: 'reply-1',
|
||||
message: 'Agent answer',
|
||||
creator: null,
|
||||
sessionId: 'session-123',
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
] as any)
|
||||
|
||||
const tool = createReadThreadTool()
|
||||
const result = await tool.execute('call-agent', { threadId: 'root-1' })
|
||||
const textContent = (result.content[0] as { text: string }).text
|
||||
|
||||
expect(textContent).toContain('- [Ai Agent] (id: reply-1, has_markup: false): Agent answer')
|
||||
})
|
||||
|
||||
it('should describe threadId parameter as referencing <thread id="..." /> in <context>', () => {
|
||||
const tool = createReadThreadTool()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const description = (tool.parameters as any).properties.threadId.description
|
||||
expect(description).toContain('<thread id="..." />')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,10 +5,24 @@ import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
const readThreadSchema = Type.Object({
|
||||
threadId: Type.String({
|
||||
description:
|
||||
'The root comment ID of a thread (e.g. from [Thread ID: ...]). Do NOT pass an asset ID or project ID.',
|
||||
'The root comment ID of a thread (from <thread id="..." /> in the <context> block). Do NOT pass an asset ID or project ID.',
|
||||
}),
|
||||
})
|
||||
|
||||
function formatCommentMeta(c: {
|
||||
id: string
|
||||
second?: number | null
|
||||
annotation?: unknown
|
||||
}): string {
|
||||
const parts = [`id: ${c.id}`]
|
||||
if (c.second !== null && c.second !== undefined) {
|
||||
parts.push(`time: ${c.second}s`)
|
||||
}
|
||||
const hasMarkup = Boolean(c.annotation && Array.isArray(c.annotation) && c.annotation.length > 0)
|
||||
parts.push(`has_markup: ${hasMarkup}`)
|
||||
return `(${parts.join(', ')})`
|
||||
}
|
||||
|
||||
export const createReadThreadTool = (): AgentTool<
|
||||
typeof readThreadSchema,
|
||||
{ threadId: string }
|
||||
@@ -47,15 +61,19 @@ export const createReadThreadTool = (): AgentTool<
|
||||
include: { creator: true },
|
||||
})
|
||||
|
||||
const rootAuthor = rootComment.creator?.name || 'User'
|
||||
let output = `Thread Root [${rootAuthor}] (${rootComment.id}): ${rootComment.message || ''}\n\nReplies (${replies.length}):`
|
||||
const isRootAgent = rootComment.creator?.type === 'agent' || rootComment.sessionId !== null
|
||||
const rootAuthor = rootComment.creator?.name || (isRootAgent ? 'Ai Agent' : 'User')
|
||||
const rootMeta = formatCommentMeta(rootComment)
|
||||
let output = `Thread Root [${rootAuthor}] ${rootMeta}: ${rootComment.message || ''}\n\nReplies (${replies.length}):`
|
||||
|
||||
if (replies.length === 0) {
|
||||
output += '\n(No replies in this thread yet)'
|
||||
} else {
|
||||
for (const reply of replies) {
|
||||
const author = reply.creator?.name || 'User'
|
||||
output += `\n- [${author}]: ${reply.message || ''}`
|
||||
const isAgent = reply.creator?.type === 'agent' || reply.sessionId !== null
|
||||
const author = reply.creator?.name || (isAgent ? 'Ai Agent' : 'User')
|
||||
const replyMeta = formatCommentMeta(reply)
|
||||
output += `\n- [${author}] ${replyMeta}: ${reply.message || ''}`
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createRenameAssetTool } from './rename-asset'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import { AuditAction, type AssetInfo } from '@shumai/dtos'
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
Permission: { Edit: 'edit', Read: 'read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/asset/asset', () => ({
|
||||
assetService: {
|
||||
updateAssetName: vi.fn().mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
name: 'Renamed Asset',
|
||||
type: 'file',
|
||||
sizeByte: 1024,
|
||||
projectId: 'proj-1',
|
||||
}),
|
||||
getAssetContext: vi.fn().mockResolvedValue({
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
auditLogService: {
|
||||
logAction: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('createRenameAssetTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('checks edit permission and calls assetService.updateAssetName', async () => {
|
||||
const tool = createRenameAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', name: 'Renamed Asset' })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'edit',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(assetService.updateAssetName).toHaveBeenCalledWith({
|
||||
id: 'asset-1',
|
||||
name: 'Renamed Asset',
|
||||
})
|
||||
|
||||
expect(result.details).toEqual({})
|
||||
expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({
|
||||
id: 'asset-1',
|
||||
name: 'Renamed Asset',
|
||||
type: 'file',
|
||||
size: 1024,
|
||||
})
|
||||
})
|
||||
|
||||
it('logs file_update audit action when renaming a file with agentContext', async () => {
|
||||
const tool = createRenameAssetTool('user-1', { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', { assetId: 'asset-1', name: 'Renamed File' })
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.file_update,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: 'proj-1',
|
||||
itemId: 'asset-1',
|
||||
})
|
||||
})
|
||||
|
||||
it('logs folder_update audit action when renaming a folder with agentContext', async () => {
|
||||
vi.mocked(assetService.updateAssetName).mockResolvedValueOnce({
|
||||
id: 'folder-1',
|
||||
name: 'Renamed Folder',
|
||||
type: 'folder',
|
||||
sizeByte: 0,
|
||||
projectId: 'proj-1',
|
||||
} as unknown as AssetInfo)
|
||||
|
||||
const tool = createRenameAssetTool('user-1', { teamId: 'team-1', agentId: 'agent-1' })
|
||||
await tool.execute('call-1', { assetId: 'folder-1', name: 'Renamed Folder' })
|
||||
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: AuditAction.folder_update,
|
||||
teamId: 'team-1',
|
||||
userId: 'user-1',
|
||||
agentId: 'agent-1',
|
||||
projectId: 'proj-1',
|
||||
itemId: 'folder-1',
|
||||
})
|
||||
})
|
||||
|
||||
it('fails when permission check rejects', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValueOnce(new Error('Permission denied'))
|
||||
const tool = createRenameAssetTool('user-1')
|
||||
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', name: 'New' })).rejects.toThrow(
|
||||
'Permission denied',
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { type User } from '@shumai/db'
|
||||
import { AuditAction } from '@shumai/dtos'
|
||||
|
||||
const renameAssetSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'The ID of the asset (file or folder) to rename. This parameter is required.',
|
||||
}),
|
||||
name: Type.String({
|
||||
description: 'The new name for the asset. This parameter is required.',
|
||||
}),
|
||||
})
|
||||
|
||||
export function createRenameAssetTool(
|
||||
userId: string,
|
||||
agentContext?: { teamId?: string; agentId?: string },
|
||||
): AgentTool<typeof renameAssetSchema> {
|
||||
return {
|
||||
name: 'rename_asset',
|
||||
label: 'Rename Asset',
|
||||
description: 'Rename an existing asset (file or folder).',
|
||||
parameters: renameAssetSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Edit,
|
||||
type: ResourceType.Asset,
|
||||
id: params.assetId,
|
||||
})
|
||||
|
||||
const updatedAsset = await assetService.updateAssetName({
|
||||
id: params.assetId,
|
||||
name: params.name,
|
||||
})
|
||||
|
||||
if (agentContext?.teamId) {
|
||||
const ctx = await assetService.getAssetContext(params.assetId).catch(() => ({
|
||||
teamId: agentContext.teamId!,
|
||||
projectId: updatedAsset.projectId ?? undefined,
|
||||
}))
|
||||
await auditLogService.logAction({
|
||||
action:
|
||||
updatedAsset.type === 'folder' ? AuditAction.folder_update : AuditAction.file_update,
|
||||
teamId: agentContext.teamId || ctx.teamId,
|
||||
userId,
|
||||
agentId: agentContext.agentId,
|
||||
projectId: ctx.projectId ?? updatedAsset.projectId ?? undefined,
|
||||
itemId: params.assetId,
|
||||
})
|
||||
}
|
||||
|
||||
const result = {
|
||||
id: updatedAsset.id,
|
||||
name: updatedAsset.name,
|
||||
type: updatedAsset.type,
|
||||
size: Number(updatedAsset.sizeByte),
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: {},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { createScreenshotTool } from './screenshot'
|
||||
import {
|
||||
prisma,
|
||||
WorkflowTaskStatus,
|
||||
type Asset,
|
||||
type AssetComment,
|
||||
type User,
|
||||
type WorkflowTask,
|
||||
} from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
return {
|
||||
...actual,
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
asset: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
assetComment: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
workflowTask: {
|
||||
create: vi.fn(),
|
||||
},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
getObject: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/workflow-core', () => ({
|
||||
workflowService: {
|
||||
executeWait: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz', () => ({
|
||||
authzService: {
|
||||
hasPermission: vi.fn(),
|
||||
},
|
||||
Permission: { Read: 'Read' },
|
||||
ResourceType: { Asset: 'asset' },
|
||||
}))
|
||||
|
||||
describe('screenshotTool', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should trigger screenshot transcode workflow and return image outputs', async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'comment-1',
|
||||
assetId: 'asset-1',
|
||||
second: 5.0,
|
||||
annotation: [
|
||||
{
|
||||
type: 'line',
|
||||
color: '#00ff00',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
} as unknown as AssetComment)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-1',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
screenshots: [
|
||||
{ key: 'screenshots/shot1.webp', timestamp: 0.0 },
|
||||
{ key: 'screenshots/shot2.webp', timestamp: 5.0 },
|
||||
],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockImplementation(async (_bucket: string, key: string) => {
|
||||
return {
|
||||
buffer: Buffer.from(`fake-bytes-for-${key}`),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string }
|
||||
})
|
||||
|
||||
const tool = createScreenshotTool('user-1', 'comment-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 10, count: 2 })
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'asset-1',
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: {
|
||||
assetId: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
type: 'transcode_screenshot',
|
||||
status: 'pending',
|
||||
payload: {
|
||||
projectId: 'project-1',
|
||||
screenshot: {
|
||||
start: 0,
|
||||
end: 10,
|
||||
count: 2,
|
||||
commentTimestamp: 5.0,
|
||||
annotations: [
|
||||
{
|
||||
type: 'line',
|
||||
color: '#00ff00',
|
||||
points: [
|
||||
[0, 0],
|
||||
[1, 1],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(result.content.length).toBe(2)
|
||||
expect(result.content[0].type).toBe('image')
|
||||
expect((result.content[0] as unknown as { data: string }).data).toBe(
|
||||
Buffer.from('fake-bytes-for-screenshots/shot1.webp').toString('base64'),
|
||||
)
|
||||
expect((result.content[1] as unknown as { data: string }).data).toBe(
|
||||
Buffer.from('fake-bytes-for-screenshots/shot2.webp').toString('base64'),
|
||||
)
|
||||
expect(result.details.sourceKeys).toEqual(['screenshots/shot1.webp', 'screenshots/shot2.webp'])
|
||||
})
|
||||
})
|
||||
@@ -1,123 +0,0 @@
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
const screenshotSchema = Type.Object({
|
||||
assetId: Type.String({
|
||||
description: 'The asset ID of the video asset. This parameter is required.',
|
||||
}),
|
||||
start: Type.Number({ description: 'Start time in seconds' }),
|
||||
end: Type.Number({ description: 'End time in seconds' }),
|
||||
count: Type.Number({ description: 'Number of screenshots to take' }),
|
||||
})
|
||||
|
||||
export function createScreenshotTool(
|
||||
userId: string,
|
||||
userCommentId?: string | null,
|
||||
): AgentTool<typeof screenshotSchema> {
|
||||
return {
|
||||
name: 'screenshot',
|
||||
label: 'Take Video Screenshots',
|
||||
description:
|
||||
'Takes screenshots for a video at specified interval. It returns the images to the agent context.',
|
||||
parameters: screenshotSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const assetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for timestamp and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === assetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger transcode workflow to take screenshots
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
screenshot: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
count: params.count,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const screenshots = output?.screenshots as
|
||||
| Array<{ key: string; timestamp: number }>
|
||||
| undefined
|
||||
|
||||
if (!screenshots || screenshots.length === 0) {
|
||||
throw new Error('No screenshots were generated.')
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const shot of screenshots) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, shot.key)
|
||||
const mimeType =
|
||||
contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp'
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType,
|
||||
})
|
||||
sourceKeys.push(shot.key)
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -735,4 +735,75 @@ describe('Agent Chat Workflow', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should include annotations and second in direct chat messageContext', async () => {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
agent: {
|
||||
prompt: 'Look at the highlighted zone',
|
||||
agentId: 'b1',
|
||||
sessionId: 'session-markup-1',
|
||||
second: 14.2,
|
||||
annotations: [{ type: 'rectangle', x: 20, y: 30, width: 40, height: 50 }],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await agentChat(task)
|
||||
|
||||
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
messageContext: expect.objectContaining({
|
||||
position: { type: 'time', seconds: 14.2 },
|
||||
annotations: [{ type: 'rectangle', x: 20, y: 30, width: 40, height: 50 }],
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should include userCommentId as id in comment messageContext', async () => {
|
||||
mockActivities.getCommentActivity.mockResolvedValue({
|
||||
id: 'comment-with-id-123',
|
||||
creatorId: 'user-c1',
|
||||
message: 'Check this out',
|
||||
replyToId: null,
|
||||
second: 5.0,
|
||||
annotation: [{ type: 'arrow' }],
|
||||
attachments: [],
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
agent: {
|
||||
agentId: 'b1',
|
||||
userCommentId: 'comment-with-id-123',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await agentChat(task)
|
||||
|
||||
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
messageContext: expect.objectContaining({
|
||||
id: 'comment-with-id-123',
|
||||
position: { type: 'time', seconds: 5.0 },
|
||||
annotation: true,
|
||||
annotations: [{ type: 'arrow' }],
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -61,6 +61,8 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
let commentAnnotation = false
|
||||
let commentCreatorId: string | undefined
|
||||
|
||||
let commentAnnotations: unknown[] | undefined
|
||||
|
||||
if (userCommentId) {
|
||||
userComment = await executeActivity(agentWorkerQueue, getCommentActivity, userCommentId)
|
||||
if (!userComment) {
|
||||
@@ -77,6 +79,7 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
userComment.annotation.length > 0
|
||||
) {
|
||||
commentAnnotation = true
|
||||
commentAnnotations = userComment.annotation
|
||||
}
|
||||
|
||||
// 2. Create Placeholder Comment
|
||||
@@ -101,6 +104,17 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
} else {
|
||||
prompt = payload.agent?.prompt || ''
|
||||
attachmentImageUrls = payload.agent?.imageUrls || []
|
||||
if (payload.agent?.second !== null && payload.agent?.second !== undefined) {
|
||||
commentTimestamp = payload.agent.second
|
||||
}
|
||||
if (
|
||||
payload.agent?.annotations &&
|
||||
Array.isArray(payload.agent.annotations) &&
|
||||
payload.agent.annotations.length > 0
|
||||
) {
|
||||
commentAnnotation = true
|
||||
commentAnnotations = payload.agent.annotations
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Get Asset
|
||||
@@ -181,6 +195,7 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
mediaType: attProxyType as ShumaiAttachedFileContext['mediaType'],
|
||||
mimeType: att.asset.mediaType || undefined,
|
||||
path: filePath || undefined,
|
||||
url: (att.asset as { url?: string }).url || undefined,
|
||||
})
|
||||
attachedAssets.push({ id: att.asset.id, name: att.asset.name, type: att.asset.type })
|
||||
}
|
||||
@@ -205,6 +220,7 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
mediaType: fileProxyType as ShumaiAttachedFileContext['mediaType'],
|
||||
mimeType: file.mediaType || undefined,
|
||||
path: filePath || undefined,
|
||||
url: (file as { url?: string }).url || undefined,
|
||||
})
|
||||
attachedAssets.push({ id: file.id, name: file.name, type: file.type })
|
||||
}
|
||||
@@ -233,6 +249,7 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
mediaType: refProxyType as ShumaiAttachedFileContext['mediaType'],
|
||||
mimeType: referencedAsset.mediaType || undefined,
|
||||
path: filePath || undefined,
|
||||
url: (referencedAsset as { url?: string }).url || undefined,
|
||||
})
|
||||
attachedAssets.push({
|
||||
id: referencedAsset.id,
|
||||
@@ -281,12 +298,18 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
|
||||
// Build structured ShumaiMessageContext
|
||||
const messageContext: ShumaiMessageContext = {
|
||||
...(userCommentId ? { id: userCommentId } : {}),
|
||||
...(userInfo
|
||||
? { user: { id: targetUserId!, name: userInfo.name, role: userInfo.role } }
|
||||
: {}),
|
||||
currentAsset: currentAssetContext,
|
||||
...(mediaPosition ? { position: mediaPosition } : {}),
|
||||
...(commentAnnotation ? { annotation: true } : {}),
|
||||
...(commentAnnotation
|
||||
? {
|
||||
annotation: true,
|
||||
annotations: commentAnnotations as ShumaiMessageContext['annotations'],
|
||||
}
|
||||
: {}),
|
||||
...(attachedFiles.length > 0 ? { attachedFiles } : {}),
|
||||
...(referencedAssets.length > 0 ? { referencedAssets } : {}),
|
||||
}
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"dependencies": {
|
||||
"@hono/zod-validator": "0.7.6",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"hono": "4.12.23",
|
||||
"@hono/zod-validator": "0.7.6",
|
||||
"zod": "4.4.3",
|
||||
"hono": "4.13.5",
|
||||
"pino": "10.3.1",
|
||||
"ulid": "3.0.2"
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.5.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +148,48 @@ describe('Chat API', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('accepts chat request with annotations, second, and attachmentIds', async () => {
|
||||
vi.mocked(chatService.startOrContinueChat).mockResolvedValue({
|
||||
sessionId: 'session1',
|
||||
taskId: 'task1',
|
||||
})
|
||||
|
||||
vi.mocked(chatService.getNewSessionMessages).mockResolvedValue({
|
||||
messages: [],
|
||||
lastEntryId: null,
|
||||
})
|
||||
|
||||
vi.mocked(chatService.getChatWorkflowStatus).mockResolvedValue({
|
||||
status: 'completed',
|
||||
output: {},
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/team1/chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
agentId: 'agent1',
|
||||
textPrompt: 'Review annotation',
|
||||
attachedFiles: ['att1'],
|
||||
annotations: [{ type: 'rectangle', x: 10, y: 20 }],
|
||||
second: 5.5,
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(chatService.startOrContinueChat).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'user1' }),
|
||||
'team1',
|
||||
expect.objectContaining({
|
||||
agentId: 'agent1',
|
||||
textPrompt: 'Review annotation',
|
||||
attachedFiles: ['att1'],
|
||||
annotations: [{ type: 'rectangle', x: 10, y: 20 }],
|
||||
second: 5.5,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('sends ping keep-alive SSE events when there are no new messages', async () => {
|
||||
vi.mocked(chatService.startOrContinueChat).mockResolvedValue({
|
||||
sessionId: 'session1',
|
||||
|
||||
@@ -26,6 +26,7 @@ import teamRoute from './team'
|
||||
import uploadRoute, { localUploadRoute } from './upload'
|
||||
import versionStackRoute from './versionStack'
|
||||
import kanbanRoute from './kanban'
|
||||
import mediaGenerationRoute from './media-generation'
|
||||
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
|
||||
@@ -69,6 +70,7 @@ const apiRoute = new Hono<{ Variables: { user: User } }>()
|
||||
.route('/', watermarkTemplateRoute)
|
||||
.route('/', quotaRoute)
|
||||
.route('/', kanbanRoute)
|
||||
.route('/', mediaGenerationRoute)
|
||||
|
||||
app.route('/files', s3Route)
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it, vi, beforeEach } from 'vitest'
|
||||
import { Hono } from 'hono'
|
||||
import mediaGenerationRoute from './media-generation'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
|
||||
vi.mock('@shumai/core/src/authz/authz')
|
||||
vi.mock('@shumai/core/src/media-generation/media-generation')
|
||||
|
||||
describe('media-generation api', () => {
|
||||
const app = new Hono<{ Variables: { user: { id: string; name: string } } }>()
|
||||
.use('*', async (c, next) => {
|
||||
c.set('user', { id: 'user1', name: 'Test User' })
|
||||
await next()
|
||||
})
|
||||
.route('/', mediaGenerationRoute)
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/media-generation returns settings', async () => {
|
||||
vi.mocked(mediaGenerationService.getSettings).mockResolvedValue({
|
||||
providers: [
|
||||
{
|
||||
provider: 'openai',
|
||||
defaultEnvKey: 'OPENAI_API_KEY',
|
||||
apiKeyConfigured: true,
|
||||
status: 'configured_env',
|
||||
supportedTypes: ['image'],
|
||||
},
|
||||
],
|
||||
enabledModels: [],
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/media-generation')
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.providers).toHaveLength(1)
|
||||
expect(data.providers[0].provider).toBe('openai')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
})
|
||||
expect(mediaGenerationService.getSettings).toHaveBeenCalledWith('t1')
|
||||
})
|
||||
|
||||
it('PUT /teams/:teamId/media-generation/providers/:provider updates api key and models', async () => {
|
||||
vi.mocked(mediaGenerationService.updateProvider).mockResolvedValue(undefined)
|
||||
|
||||
const res = await app.request('/teams/t1/media-generation/providers/openai', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
apiKey: 'sk-new-key',
|
||||
models: [{ type: 'image', modelId: 'dall-e-3', name: 'DALL-E 3' }],
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.success).toBe(true)
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
})
|
||||
expect(mediaGenerationService.updateProvider).toHaveBeenCalledWith('t1', 'openai', {
|
||||
apiKey: 'sk-new-key',
|
||||
models: [{ type: 'image', modelId: 'dall-e-3', name: 'DALL-E 3' }],
|
||||
})
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/media-generation/models adds an enabled model', async () => {
|
||||
vi.mocked(mediaGenerationService.addEnabledModel).mockResolvedValue({
|
||||
id: 'm1',
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3',
|
||||
createdAt: new Date().toISOString(),
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/media-generation/models', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(201)
|
||||
const data = await res.json()
|
||||
expect(data.id).toBe('m1')
|
||||
expect(data.modelId).toBe('dall-e-3')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
})
|
||||
expect(mediaGenerationService.addEnabledModel).toHaveBeenCalledWith('t1', {
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3',
|
||||
})
|
||||
})
|
||||
|
||||
it('DELETE /teams/:teamId/media-generation/models/:modelId removes an enabled model', async () => {
|
||||
vi.mocked(mediaGenerationService.removeEnabledModel).mockResolvedValue(undefined)
|
||||
|
||||
const res = await app.request('/teams/t1/media-generation/models/m1', {
|
||||
method: 'DELETE',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.success).toBe(true)
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
})
|
||||
expect(mediaGenerationService.removeEnabledModel).toHaveBeenCalledWith('t1', 'm1')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,97 @@
|
||||
import { zValidator } from '@hono/zod-validator'
|
||||
import { Hono } from 'hono'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { mediaGenerationService } from '@shumai/core/src/media-generation/media-generation'
|
||||
import {
|
||||
createEnabledMediaModelSchema,
|
||||
getCuratedMediaModelsQuerySchema,
|
||||
updateMediaProviderApiKeySchema,
|
||||
} from '@shumai/dtos'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
|
||||
const route = new Hono<{ Variables: { user: User } }>()
|
||||
.get('/teams/:teamId/media-generation', async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const user = c.get('user')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const settings = await mediaGenerationService.getSettings(teamId)
|
||||
return c.json(settings)
|
||||
})
|
||||
|
||||
.put(
|
||||
'/teams/:teamId/media-generation/providers/:provider',
|
||||
zValidator('json', updateMediaProviderApiKeySchema),
|
||||
async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const provider = c.req.param('provider')
|
||||
const user = c.get('user')
|
||||
const body = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
await mediaGenerationService.updateProvider(teamId, provider, body)
|
||||
return c.json({ success: true })
|
||||
},
|
||||
)
|
||||
|
||||
.post(
|
||||
'/teams/:teamId/media-generation/models',
|
||||
zValidator('json', createEnabledMediaModelSchema),
|
||||
async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const user = c.get('user')
|
||||
const body = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const model = await mediaGenerationService.addEnabledModel(teamId, body)
|
||||
return c.json(model, 201)
|
||||
},
|
||||
)
|
||||
|
||||
.get(
|
||||
'/teams/:teamId/media-generation/models/curated',
|
||||
zValidator('query', getCuratedMediaModelsQuerySchema),
|
||||
async (c) => {
|
||||
const { provider, type } = c.req.valid('query')
|
||||
const models = mediaGenerationService.getCuratedModels(provider, type)
|
||||
return c.json(models)
|
||||
},
|
||||
)
|
||||
|
||||
.delete('/teams/:teamId/media-generation/models/:modelId', async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const modelId = c.req.param('modelId')
|
||||
const user = c.get('user')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
await mediaGenerationService.removeEnabledModel(teamId, modelId)
|
||||
return c.json({ success: true })
|
||||
})
|
||||
|
||||
export default route
|
||||
@@ -163,13 +163,13 @@ describe('provider api', () => {
|
||||
const res = await app.request('/providers/p1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ config, models }),
|
||||
body: JSON.stringify({ name: 'new-name', config, models }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.config.baseUrl).toBe('https://new-url.com')
|
||||
expect(providerService.update).toHaveBeenCalledWith('t1', 'p1', config, models)
|
||||
expect(providerService.update).toHaveBeenCalledWith('t1', 'p1', config, models, 'new-name')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
@@ -184,6 +184,111 @@ describe('provider api', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('POST /providers/:id/models creates a model', async () => {
|
||||
vi.mocked(providerService.getById).mockResolvedValue({
|
||||
id: 'p1',
|
||||
teamId: 't1',
|
||||
} as unknown as Awaited<ReturnType<typeof providerService.getById>>)
|
||||
const modelData = {
|
||||
modelId: 'm1',
|
||||
name: 'Model 1',
|
||||
config: {
|
||||
reasoning: true,
|
||||
input: ['text' as const],
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
}
|
||||
vi.mocked(providerService.createModel).mockResolvedValue({
|
||||
id: 'm-db-1',
|
||||
providerId: 'p1',
|
||||
modelId: 'm1',
|
||||
name: 'Model 1',
|
||||
config: modelData.config,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
|
||||
const res = await app.request('/providers/p1/models', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(modelData),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(201)
|
||||
const data = await res.json()
|
||||
expect(data.id).toBe('m-db-1')
|
||||
expect(providerService.createModel).toHaveBeenCalledWith('t1', 'p1', modelData)
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Provider,
|
||||
id: 'p1',
|
||||
})
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: 'provider_update',
|
||||
teamId: 't1',
|
||||
userId: 'user1',
|
||||
itemId: 'p1',
|
||||
})
|
||||
})
|
||||
|
||||
it('PUT /providers/:id/models/:modelDbId updates a model', async () => {
|
||||
vi.mocked(providerService.getById).mockResolvedValue({
|
||||
id: 'p1',
|
||||
teamId: 't1',
|
||||
} as unknown as Awaited<ReturnType<typeof providerService.getById>>)
|
||||
const updateData = {
|
||||
name: 'Updated Model',
|
||||
}
|
||||
vi.mocked(providerService.updateModel).mockResolvedValue({
|
||||
id: 'm-db-1',
|
||||
providerId: 'p1',
|
||||
modelId: 'm1',
|
||||
name: 'Updated Model',
|
||||
config: {
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
|
||||
const res = await app.request('/providers/p1/models/m-db-1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(updateData),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.name).toBe('Updated Model')
|
||||
expect(providerService.updateModel).toHaveBeenCalledWith('t1', 'p1', 'm-db-1', updateData)
|
||||
})
|
||||
|
||||
it('DELETE /providers/:id/models/:modelDbId deletes a model', async () => {
|
||||
vi.mocked(providerService.getById).mockResolvedValue({
|
||||
id: 'p1',
|
||||
teamId: 't1',
|
||||
} as unknown as Awaited<ReturnType<typeof providerService.getById>>)
|
||||
vi.mocked(providerService.deleteModel).mockResolvedValue({
|
||||
id: 'm-db-1',
|
||||
} as unknown as Awaited<ReturnType<typeof providerService.deleteModel>>)
|
||||
|
||||
const res = await app.request('/providers/p1/models/m-db-1', {
|
||||
method: 'DELETE',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.success).toBe(true)
|
||||
expect(providerService.deleteModel).toHaveBeenCalledWith('t1', 'p1', 'm-db-1')
|
||||
})
|
||||
|
||||
it('DELETE /providers/:id deletes provider', async () => {
|
||||
vi.mocked(providerService.getById).mockResolvedValue({
|
||||
id: 'p1',
|
||||
@@ -214,4 +319,89 @@ describe('provider api', () => {
|
||||
itemId: 'p1',
|
||||
})
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/providers/sync-check checks for updates', async () => {
|
||||
vi.mocked(providerService.checkUpdates).mockResolvedValue({
|
||||
providers: [
|
||||
{
|
||||
name: 'deepseek',
|
||||
isNewProvider: true,
|
||||
config: { api: 'openai-responses', apiKey: 'DEEPSEEK_API_KEY' },
|
||||
models: [],
|
||||
},
|
||||
],
|
||||
totalNewProviders: 1,
|
||||
totalNewModels: 0,
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/providers/sync-check', {
|
||||
method: 'POST',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.totalNewProviders).toBe(1)
|
||||
expect(data.providers).toHaveLength(1)
|
||||
expect(providerService.checkUpdates).toHaveBeenCalledWith('t1')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
})
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/providers/sync-apply applies sync selection', async () => {
|
||||
vi.mocked(providerService.applySync).mockResolvedValue({
|
||||
addedProviders: 1,
|
||||
addedModels: 3,
|
||||
})
|
||||
|
||||
const payload = {
|
||||
providers: [
|
||||
{
|
||||
name: 'deepseek',
|
||||
isNewProvider: true,
|
||||
config: { api: 'openai-responses', apiKey: 'KEY' },
|
||||
models: [
|
||||
{
|
||||
modelId: 'deepseek-v4-flash',
|
||||
name: 'DeepSeek V4 Flash',
|
||||
config: {
|
||||
api: 'openai-responses',
|
||||
reasoning: true,
|
||||
input: ['text'],
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const res = await app.request('/teams/t1/providers/sync-apply', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.addedProviders).toBe(1)
|
||||
expect(data.addedModels).toBe(3)
|
||||
expect(providerService.applySync).toHaveBeenCalledWith('t1', payload)
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: expect.anything(),
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
})
|
||||
expect(auditLogService.logAction).toHaveBeenCalledWith({
|
||||
action: 'provider_update',
|
||||
teamId: 't1',
|
||||
userId: 'user1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,7 +2,14 @@ import { zValidator } from '@hono/zod-validator'
|
||||
import { Hono } from 'hono'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { providerService } from '@shumai/core/src/provider/provider'
|
||||
import { createProviderRequestSchema, updateProviderRequestSchema, AuditAction } from '@shumai/dtos'
|
||||
import {
|
||||
createModelRequestSchema,
|
||||
createProviderRequestSchema,
|
||||
syncApplyRequestSchema,
|
||||
updateModelRequestSchema,
|
||||
updateProviderRequestSchema,
|
||||
AuditAction,
|
||||
} from '@shumai/dtos'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
|
||||
@@ -24,6 +31,46 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
return c.json(providers)
|
||||
})
|
||||
|
||||
.post('/teams/:teamId/providers/sync-check', async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const user = c.get('user')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const result = await providerService.checkUpdates(teamId)
|
||||
return c.json(result)
|
||||
})
|
||||
|
||||
.post(
|
||||
'/teams/:teamId/providers/sync-apply',
|
||||
zValidator('json', syncApplyRequestSchema),
|
||||
async (c) => {
|
||||
const teamId = c.req.param('teamId')
|
||||
const user = c.get('user')
|
||||
const body = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const result = await providerService.applySync(teamId, body)
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.provider_update,
|
||||
teamId,
|
||||
userId: user?.id,
|
||||
})
|
||||
return c.json(result)
|
||||
},
|
||||
)
|
||||
|
||||
.get('/providers/:id/models', async (c) => {
|
||||
const id = c.req.param('id')
|
||||
const user = c.get('user')
|
||||
@@ -67,7 +114,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
.put('/providers/:id', zValidator('json', updateProviderRequestSchema), async (c) => {
|
||||
const id = c.req.param('id')
|
||||
const user = c.get('user')
|
||||
const { config, models } = c.req.valid('json')
|
||||
const { name, config, models } = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
@@ -79,7 +126,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
const providerBefore = await providerService.getById(id)
|
||||
if (!providerBefore) throw new Error('Provider not found')
|
||||
|
||||
const provider = await providerService.update(providerBefore.teamId, id, config, models)
|
||||
const provider = await providerService.update(providerBefore.teamId, id, config, models, name)
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.provider_update,
|
||||
teamId: providerBefore.teamId,
|
||||
@@ -89,6 +136,86 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
return c.json(provider)
|
||||
})
|
||||
|
||||
.post('/providers/:id/models', zValidator('json', createModelRequestSchema), async (c) => {
|
||||
const id = c.req.param('id')
|
||||
const user = c.get('user')
|
||||
const data = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Provider,
|
||||
id,
|
||||
})
|
||||
|
||||
const provider = await providerService.getById(id)
|
||||
if (!provider) throw new Error('Provider not found')
|
||||
|
||||
const model = await providerService.createModel(provider.teamId, id, data)
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.provider_update,
|
||||
teamId: provider.teamId,
|
||||
userId: user?.id,
|
||||
itemId: id,
|
||||
})
|
||||
return c.json(model, 201)
|
||||
})
|
||||
|
||||
.put(
|
||||
'/providers/:id/models/:modelDbId',
|
||||
zValidator('json', updateModelRequestSchema),
|
||||
async (c) => {
|
||||
const id = c.req.param('id')
|
||||
const modelDbId = c.req.param('modelDbId')
|
||||
const user = c.get('user')
|
||||
const data = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Provider,
|
||||
id,
|
||||
})
|
||||
|
||||
const provider = await providerService.getById(id)
|
||||
if (!provider) throw new Error('Provider not found')
|
||||
|
||||
const model = await providerService.updateModel(provider.teamId, id, modelDbId, data)
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.provider_update,
|
||||
teamId: provider.teamId,
|
||||
userId: user?.id,
|
||||
itemId: id,
|
||||
})
|
||||
return c.json(model)
|
||||
},
|
||||
)
|
||||
|
||||
.delete('/providers/:id/models/:modelDbId', async (c) => {
|
||||
const id = c.req.param('id')
|
||||
const modelDbId = c.req.param('modelDbId')
|
||||
const user = c.get('user')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Provider,
|
||||
id,
|
||||
})
|
||||
|
||||
const providerBefore = await providerService.getById(id)
|
||||
if (!providerBefore) throw new Error('Provider not found')
|
||||
|
||||
await providerService.deleteModel(providerBefore.teamId, id, modelDbId)
|
||||
await auditLogService.logAction({
|
||||
action: AuditAction.provider_update,
|
||||
teamId: providerBefore.teamId,
|
||||
userId: user?.id,
|
||||
itemId: id,
|
||||
})
|
||||
return c.json({ success: true })
|
||||
})
|
||||
|
||||
.delete('/providers/:id', async (c) => {
|
||||
const id = c.req.param('id')
|
||||
const user = c.get('user')
|
||||
|
||||
@@ -513,6 +513,8 @@ describe('team api', () => {
|
||||
action: AuditAction.project_create,
|
||||
teamId: 't1',
|
||||
userId: 'user1',
|
||||
agentId: null,
|
||||
agent: null,
|
||||
projectId: 'p1',
|
||||
itemId: 'p1',
|
||||
createdAt: new Date().toISOString(),
|
||||
|
||||
@@ -391,6 +391,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
teamId,
|
||||
actions: query.actions,
|
||||
userIds: query.userIds,
|
||||
agentIds: query.agentIds,
|
||||
itemId: query.itemId,
|
||||
first: query.first,
|
||||
after: query.after,
|
||||
|
||||
+29
-10
@@ -1,34 +1,53 @@
|
||||
{
|
||||
"name": "@shumai/core",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"dependencies": {
|
||||
"@ai-sdk/alibaba": "2.0.38",
|
||||
"@ai-sdk/amazon-bedrock": "5.0.67",
|
||||
"@ai-sdk/azure": "4.0.53",
|
||||
"@ai-sdk/black-forest-labs": "2.0.34",
|
||||
"@ai-sdk/bytedance": "2.0.36",
|
||||
"@ai-sdk/deepinfra": "3.0.40",
|
||||
"@ai-sdk/fal": "3.0.34",
|
||||
"@ai-sdk/fireworks": "3.0.43",
|
||||
"@ai-sdk/google": "4.0.57",
|
||||
"@ai-sdk/google-vertex": "5.0.69",
|
||||
"@ai-sdk/klingai": "4.0.35",
|
||||
"@ai-sdk/luma": "3.0.34",
|
||||
"@ai-sdk/minimax": "3.0.21",
|
||||
"@ai-sdk/openai": "4.0.51",
|
||||
"@ai-sdk/prodia": "2.0.34",
|
||||
"@ai-sdk/replicate": "3.0.34",
|
||||
"@ai-sdk/togetherai": "3.0.41",
|
||||
"@ai-sdk/xai": "4.0.49",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@earendil-works/pi-ai": "0.82.1",
|
||||
"@modelcontextprotocol/client": "2.0.0",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@sinclair/typebox": "0.34.49",
|
||||
"adm-zip": "0.5.17",
|
||||
"@sinclair/typebox": "0.34.52",
|
||||
"adm-zip": "0.6.0",
|
||||
"ai": "7.0.84",
|
||||
"better-auth": "1.6.11",
|
||||
"csv-parse": "7.0.1",
|
||||
"csv-parse": "7.0.2",
|
||||
"date-fns": "4.4.0",
|
||||
"dotenv": "16.6.1",
|
||||
"dotenv": "17.4.2",
|
||||
"dotenv-expand": "13.0.0",
|
||||
"extract-zip": "2.0.1",
|
||||
"hono": "4.12.23",
|
||||
"hono": "4.13.5",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"pdfkit": "0.19.1",
|
||||
"pdfkit": "0.20.1",
|
||||
"pino": "10.3.1",
|
||||
"sharp": "0.34.5",
|
||||
"sharp": "0.35.4",
|
||||
"tweetnacl": "1.0.3",
|
||||
"typebox": "1.1.38",
|
||||
"typebox": "1.3.22",
|
||||
"ulid": "3.0.2",
|
||||
"yaml": "2.9.0",
|
||||
"zod": "4.4.3"
|
||||
"zod": "4.5.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@modelcontextprotocol/server": "2.0.0",
|
||||
|
||||
@@ -257,6 +257,169 @@ describe('AgentService', () => {
|
||||
expect(updated.user.image).toBe('files/01JK23456789ABCDEF01234567')
|
||||
})
|
||||
|
||||
test('Update Agent preserves deniedTools when omitted, and allows explicit empty array', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team, provider, model, agent: legacyAgent } = await setupTestData(db)
|
||||
|
||||
// Legacy agent has no deniedTools in config, update without deniedTools should fall back to DEFAULT_DENIED_TOOLS
|
||||
const updatedLegacy = await svc.updateAgent({
|
||||
agentId: legacyAgent.id,
|
||||
name: 'Updated Legacy',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: legacyAgent.providerId!,
|
||||
modelId: legacyAgent.modelId!,
|
||||
thinkingLevel: 'medium',
|
||||
})
|
||||
expect((updatedLegacy.config as PrismaJson.AgentConfig).deniedTools).toEqual(['delete_asset'])
|
||||
|
||||
// Create new agent with default deniedTools
|
||||
const agent = await svc.createAgent({
|
||||
teamId: team.id,
|
||||
name: 'New Agent with Defaults',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
thinkingLevel: 'medium',
|
||||
})
|
||||
|
||||
expect(agent).not.toBeNull()
|
||||
|
||||
// Agent created with default deniedTools: ['delete_asset']
|
||||
const createdConfig = agent!.config as PrismaJson.AgentConfig
|
||||
expect(createdConfig.deniedTools).toEqual(['delete_asset'])
|
||||
|
||||
// Update without deniedTools: should preserve ['delete_asset']
|
||||
const updatedWithoutDenied = await svc.updateAgent({
|
||||
agentId: agent!.id,
|
||||
name: 'Updated Name',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: agent!.providerId!,
|
||||
modelId: agent!.modelId!,
|
||||
thinkingLevel: 'medium',
|
||||
})
|
||||
const config1 = updatedWithoutDenied.config as PrismaJson.AgentConfig
|
||||
expect(config1.deniedTools).toEqual(['delete_asset'])
|
||||
|
||||
// Update with explicit deniedTools: []
|
||||
const updatedWithEmptyDenied = await svc.updateAgent({
|
||||
agentId: agent!.id,
|
||||
name: 'Updated Name 2',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
providerId: agent!.providerId!,
|
||||
modelId: agent!.modelId!,
|
||||
thinkingLevel: 'medium',
|
||||
deniedTools: [],
|
||||
})
|
||||
const config2 = updatedWithEmptyDenied.config as PrismaJson.AgentConfig
|
||||
expect(config2.deniedTools).toEqual([])
|
||||
})
|
||||
|
||||
test('Create and Update Agent with Preset Avatars', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team, provider, model } = await setupTestData(db)
|
||||
|
||||
// 1. Create with specific preset avatar
|
||||
const agent2 = await svc.createAgent({
|
||||
teamId: team.id,
|
||||
name: 'Agent Preset 2',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
thinkingLevel: 'off',
|
||||
avatar: 'avatar-2',
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
})
|
||||
expect(agent2!.user.image).toMatch(/^files\/[A-Z0-9]{26}\.webp$/)
|
||||
expect((agent2!.config as unknown as PrismaJson.AgentConfig).avatarPreset).toBe('avatar-2')
|
||||
const initialKey = agent2!.user.image
|
||||
|
||||
// 2. Create without avatar (should default to avatar-1)
|
||||
const agentDefault = await svc.createAgent({
|
||||
teamId: team.id,
|
||||
name: 'Agent Default',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
thinkingLevel: 'off',
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
})
|
||||
expect(agentDefault!.user.image).toMatch(/^files\/[A-Z0-9]{26}\.webp$/)
|
||||
expect((agentDefault!.config as unknown as PrismaJson.AgentConfig).avatarPreset).toBe(
|
||||
'avatar-1',
|
||||
)
|
||||
|
||||
const agentInfo = await svc.toAgentInfo(agentDefault!)
|
||||
expect(agentInfo.avatarPreset).toBe('avatar-1')
|
||||
|
||||
// 3. Update agent with a different preset avatar
|
||||
const updatedPreset = await svc.updateAgent({
|
||||
agentId: agent2!.id,
|
||||
name: 'Agent Preset 3',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
thinkingLevel: 'off',
|
||||
avatar: 'avatar-3',
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
})
|
||||
expect(updatedPreset.user.image).toMatch(/^files\/[A-Z0-9]{26}\.webp$/)
|
||||
expect(updatedPreset.user.image).not.toBe(initialKey)
|
||||
expect((updatedPreset.config as unknown as PrismaJson.AgentConfig).avatarPreset).toBe(
|
||||
'avatar-3',
|
||||
)
|
||||
|
||||
// 3b. Update agent with the SAME preset avatar (should preserve image key without re-upload)
|
||||
const keyBeforeSameUpdate = updatedPreset.user.image
|
||||
const updatedSamePreset = await svc.updateAgent({
|
||||
agentId: agent2!.id,
|
||||
name: 'Agent Same Preset 3',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
thinkingLevel: 'off',
|
||||
avatar: 'avatar-3',
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
})
|
||||
expect(updatedSamePreset.user.image).toBe(keyBeforeSameUpdate)
|
||||
expect((updatedSamePreset.config as unknown as PrismaJson.AgentConfig).avatarPreset).toBe(
|
||||
'avatar-3',
|
||||
)
|
||||
|
||||
// 4. Update with presigned URL containing .webp extension
|
||||
const updatedPresigned = await svc.updateAgent({
|
||||
agentId: agent2!.id,
|
||||
name: 'Agent Presigned',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
thinkingLevel: 'off',
|
||||
avatar: 'https://shumai.s3.amazonaws.com/files/01JK23456789ABCDEF01234567.webp?foo=bar',
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
})
|
||||
expect(updatedPresigned.user.image).toBe('files/01JK23456789ABCDEF01234567.webp')
|
||||
expect(
|
||||
(updatedPresigned.config as unknown as PrismaJson.AgentConfig).avatarPreset,
|
||||
).toBeUndefined()
|
||||
|
||||
// 5. Update without avatar specified (avatar undefined) - preserves image
|
||||
const updatedWithoutAvatar = await svc.updateAgent({
|
||||
agentId: agent2!.id,
|
||||
name: 'Agent Without Avatar Change',
|
||||
type: 'chat',
|
||||
enabled: true,
|
||||
thinkingLevel: 'off',
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
})
|
||||
expect(updatedWithoutAvatar.user.image).toBe('files/01JK23456789ABCDEF01234567.webp')
|
||||
})
|
||||
|
||||
test('Delete Agent', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
@@ -338,10 +501,89 @@ describe('AgentService', () => {
|
||||
expect(entries[0].id).toBe(entry1.id)
|
||||
expect(entries[1].id).toBe(entry2.id)
|
||||
})
|
||||
|
||||
test('getSessionEntries dynamically attaches details.thread on custom_message entries with replies', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { agent, team } = await setupTestData(db)
|
||||
|
||||
const project = await db.project.create({
|
||||
data: { name: 'Test Project', teamId: team.id },
|
||||
})
|
||||
const asset = await db.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
const user = await db.user.create({
|
||||
data: { name: 'Alice', email: `alice-${Date.now()}@example.com` },
|
||||
})
|
||||
|
||||
const rootComment = await db.assetComment.create({
|
||||
data: {
|
||||
id: 'comment-root-entries',
|
||||
assetId: asset.id,
|
||||
message: 'Top-level comment',
|
||||
creatorId: user.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Add a reply
|
||||
await db.assetComment.create({
|
||||
data: {
|
||||
id: 'comment-reply-entries',
|
||||
assetId: asset.id,
|
||||
message: 'Reply to root',
|
||||
creatorId: user.id,
|
||||
replyToId: rootComment.id,
|
||||
},
|
||||
})
|
||||
|
||||
const session = await db.agentSession.create({
|
||||
data: {
|
||||
agentId: agent.id,
|
||||
type: 'comment',
|
||||
cwd: '/tmp',
|
||||
},
|
||||
})
|
||||
|
||||
const entry = await db.agentSessionEntry.create({
|
||||
data: {
|
||||
id: rootComment.id,
|
||||
sessionId: session.id,
|
||||
type: 'custom_message',
|
||||
parentId: null,
|
||||
data: {
|
||||
customType: 'shumai_message',
|
||||
content: 'Top-level comment',
|
||||
display: true,
|
||||
details: { user: { id: user.id, name: user.name } },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await db.agentSession.update({
|
||||
where: { id: session.id },
|
||||
data: { leafId: entry.id },
|
||||
})
|
||||
|
||||
const entries = await svc.getSessionEntries({ sessionId: session.id })
|
||||
expect(entries).toHaveLength(1)
|
||||
expect(entries[0].id).toBe(rootComment.id)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const entryDetails = (entries[0].entry as any).details
|
||||
expect(entryDetails?.thread).toEqual({
|
||||
id: rootComment.id,
|
||||
replyCount: 1,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('listSessions', () => {
|
||||
test('lists sessions for team and excludes pending id', async () => {
|
||||
test('lists sessions for team, includes chat sessions and leaf comment sessions, hides main comment sessions and excludes pending id', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team, agent } = await setupTestData(db)
|
||||
@@ -353,27 +595,43 @@ describe('AgentService', () => {
|
||||
},
|
||||
})
|
||||
|
||||
const session1 = await db.agentSession.create({
|
||||
// 1. Chat session (userCommentId is null) - should be included!
|
||||
const chatSession = await db.agentSession.create({
|
||||
data: {
|
||||
name: 'Session One',
|
||||
name: 'Chat Session',
|
||||
agentId: agent.id,
|
||||
userId: user.id,
|
||||
type: 'chat',
|
||||
cwd: '/tmp',
|
||||
userCommentId: null,
|
||||
},
|
||||
})
|
||||
|
||||
const session2 = await db.agentSession.create({
|
||||
// 2. Main comment session (userCommentId is null) - should be HIDDEN / EXCLUDED!
|
||||
const mainCommentSession = await db.agentSession.create({
|
||||
data: {
|
||||
name: 'Session Two',
|
||||
name: 'Main Comment Session',
|
||||
agentId: agent.id,
|
||||
userId: user.id,
|
||||
type: 'comment',
|
||||
cwd: '/tmp',
|
||||
userCommentId: null,
|
||||
},
|
||||
})
|
||||
|
||||
// Pending session - should be excluded!
|
||||
// 3. Leaf comment session (userCommentId is not null) - should be included!
|
||||
const leafCommentSession = await db.agentSession.create({
|
||||
data: {
|
||||
name: 'Leaf Comment Session',
|
||||
agentId: agent.id,
|
||||
userId: user.id,
|
||||
type: 'comment',
|
||||
cwd: '/tmp',
|
||||
userCommentId: 'comment-123',
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Pending session - should be excluded!
|
||||
await db.agentSession.create({
|
||||
data: {
|
||||
id: 'pending',
|
||||
@@ -388,15 +646,21 @@ describe('AgentService', () => {
|
||||
const res = await svc.listSessions(team.id, { first: 10 })
|
||||
|
||||
expect(res.data).toHaveLength(2)
|
||||
expect(res.data.map((s) => s.id)).not.toContain('pending')
|
||||
const sessionIds = res.data.map((s) => s.id)
|
||||
expect(sessionIds).toContain(session1.id)
|
||||
expect(sessionIds).toContain(session2.id)
|
||||
expect(sessionIds).not.toContain('pending')
|
||||
expect(sessionIds).not.toContain(mainCommentSession.id)
|
||||
expect(sessionIds).toContain(chatSession.id)
|
||||
expect(sessionIds).toContain(leafCommentSession.id)
|
||||
|
||||
const found = res.data.find((s) => s.id === session1.id)
|
||||
expect(found?.name).toBe('Session One')
|
||||
expect(found?.type).toBe('chat')
|
||||
expect(found?.creator?.name).toBe('Regular User')
|
||||
const foundChat = res.data.find((s) => s.id === chatSession.id)
|
||||
expect(foundChat?.name).toBe('Chat Session')
|
||||
expect(foundChat?.type).toBe('chat')
|
||||
expect(foundChat?.creator?.name).toBe('Regular User')
|
||||
|
||||
const foundLeafComment = res.data.find((s) => s.id === leafCommentSession.id)
|
||||
expect(foundLeafComment?.name).toBe('Leaf Comment Session')
|
||||
expect(foundLeafComment?.type).toBe('comment')
|
||||
expect(foundLeafComment?.creator?.name).toBe('Regular User')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
AgentSessionInfo,
|
||||
AgentInfo,
|
||||
AgentType,
|
||||
DEFAULT_DENIED_TOOLS,
|
||||
} from '@shumai/dtos'
|
||||
import {
|
||||
paginateQuery,
|
||||
@@ -18,6 +19,8 @@ import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { getAvatarUrl } from '@shumai/core/src/user/avatar'
|
||||
import { resolveEffectiveRole } from '@shumai/core/src/authz/authz'
|
||||
import { getAllowedAgentRoles } from './permissions'
|
||||
import { ulid } from 'ulid'
|
||||
import { isPresetAvatarId, getPresetAvatarBuffer } from './presets'
|
||||
import AdmZip from 'adm-zip'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
@@ -47,7 +50,7 @@ interface AgentInfoSource {
|
||||
|
||||
function extractS3Key(urlOrKey?: string | null): string | null | undefined {
|
||||
if (!urlOrKey) return urlOrKey
|
||||
const match = urlOrKey.match(/files\/[A-Z0-9]{26}/)
|
||||
const match = urlOrKey.match(/files\/[A-Z0-9]{26}(\.[a-zA-Z0-9]+)?/)
|
||||
return match ? match[0] : urlOrKey
|
||||
}
|
||||
|
||||
@@ -178,6 +181,23 @@ export class AgentService {
|
||||
}
|
||||
}
|
||||
|
||||
const targetAvatar = avatar || 'avatar-1'
|
||||
let avatarPreset: string | undefined
|
||||
let imageKey: string | null | undefined
|
||||
|
||||
if (isPresetAvatarId(targetAvatar)) {
|
||||
avatarPreset = targetAvatar
|
||||
const buffer = getPresetAvatarBuffer(targetAvatar)
|
||||
if (buffer) {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const key = `files/${ulid()}.webp`
|
||||
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
|
||||
imageKey = key
|
||||
}
|
||||
} else {
|
||||
imageKey = extractS3Key(avatar)
|
||||
}
|
||||
|
||||
return this.prismaClient.$transaction(async (tx) => {
|
||||
const team = await tx.team.findUnique({
|
||||
where: { id: teamId },
|
||||
@@ -189,7 +209,8 @@ export class AgentService {
|
||||
model: modelId || '',
|
||||
thinkingLevel,
|
||||
systemPrompt,
|
||||
deniedTools: deniedTools || [],
|
||||
deniedTools: deniedTools ?? [...DEFAULT_DENIED_TOOLS],
|
||||
avatarPreset,
|
||||
}
|
||||
|
||||
const user = await tx.user.create({
|
||||
@@ -197,7 +218,7 @@ export class AgentService {
|
||||
name,
|
||||
email: `agent-${Date.now()}-${Math.random().toString(36).substring(7)}@shumai.ai`,
|
||||
type: 'agent',
|
||||
image: extractS3Key(avatar),
|
||||
image: imageKey,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -298,12 +319,44 @@ export class AgentService {
|
||||
}
|
||||
}
|
||||
|
||||
const currentConfig = (agent.config as unknown as PrismaJson.AgentConfig) || {}
|
||||
let newAvatarPreset: string | undefined = currentConfig.avatarPreset
|
||||
let imageKey: string | null | undefined
|
||||
|
||||
if (avatar !== undefined) {
|
||||
if (!avatar) {
|
||||
imageKey = null
|
||||
newAvatarPreset = undefined
|
||||
} else if (isPresetAvatarId(avatar)) {
|
||||
if (avatar === currentConfig.avatarPreset && agent.user.image) {
|
||||
imageKey = agent.user.image
|
||||
newAvatarPreset = avatar
|
||||
} else {
|
||||
const buffer = getPresetAvatarBuffer(avatar)
|
||||
if (buffer) {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const key = `files/${ulid()}.webp`
|
||||
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
|
||||
imageKey = key
|
||||
newAvatarPreset = avatar
|
||||
}
|
||||
}
|
||||
} else {
|
||||
imageKey = extractS3Key(avatar)
|
||||
newAvatarPreset = undefined
|
||||
}
|
||||
}
|
||||
|
||||
const agentConfig: PrismaJson.AgentConfig = {
|
||||
provider: providerId || '',
|
||||
model: modelId || '',
|
||||
thinkingLevel,
|
||||
systemPrompt,
|
||||
deniedTools: deniedTools || [],
|
||||
deniedTools:
|
||||
deniedTools !== undefined
|
||||
? deniedTools
|
||||
: (currentConfig.deniedTools ?? [...DEFAULT_DENIED_TOOLS]),
|
||||
avatarPreset: newAvatarPreset,
|
||||
}
|
||||
|
||||
return this.prismaClient.$transaction(async (tx) => {
|
||||
@@ -311,7 +364,7 @@ export class AgentService {
|
||||
where: { id: agentId },
|
||||
data: {
|
||||
name,
|
||||
image: extractS3Key(avatar),
|
||||
...(imageKey !== undefined ? { image: imageKey } : {}),
|
||||
},
|
||||
})
|
||||
|
||||
@@ -415,6 +468,7 @@ export class AgentService {
|
||||
enabled: agent.enabled,
|
||||
permission: agent.permission,
|
||||
avatar: (await getAvatarUrl(agent.user.image)) || undefined,
|
||||
avatarPreset: config.avatarPreset,
|
||||
providerId: agent.providerId || undefined,
|
||||
modelId: agent.modelId || undefined,
|
||||
thinkingLevel: config.thinkingLevel || 'off',
|
||||
@@ -549,21 +603,53 @@ export class AgentService {
|
||||
ORDER BY depth DESC;
|
||||
`
|
||||
|
||||
const entryIds = rows.map((r) => r.id)
|
||||
const threadReplyCountMap = new Map<string, number>()
|
||||
if (entryIds.length > 0) {
|
||||
const threadCounts = await this.prismaClient.assetComment.groupBy({
|
||||
by: ['replyToId'],
|
||||
where: {
|
||||
replyToId: { in: entryIds },
|
||||
message: { not: '__CHAT__' },
|
||||
},
|
||||
_count: {
|
||||
id: true,
|
||||
},
|
||||
})
|
||||
for (const tc of threadCounts) {
|
||||
if (tc.replyToId) {
|
||||
threadReplyCountMap.set(tc.replyToId, tc._count.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rows.map((r) => {
|
||||
const payload = (r.data as Record<string, unknown>) || {}
|
||||
const count = threadReplyCountMap.get(r.id)
|
||||
let details = payload.details as Record<string, unknown> | undefined
|
||||
if (count !== undefined && count > 0 && r.type === 'custom_message') {
|
||||
details = {
|
||||
...details,
|
||||
thread: {
|
||||
id: r.id,
|
||||
replyCount: count,
|
||||
},
|
||||
}
|
||||
}
|
||||
const entryObj = {
|
||||
id: r.id,
|
||||
type: r.type || 'message',
|
||||
parentId: r.parent_id,
|
||||
timestamp: r.created_at.toISOString(),
|
||||
...payload,
|
||||
...(details ? { details } : {}),
|
||||
}
|
||||
return {
|
||||
id: r.id,
|
||||
sessionId: r.session_id || params.sessionId,
|
||||
type: r.type,
|
||||
parentId: r.parent_id,
|
||||
data: r.data,
|
||||
data: details !== payload.details ? { ...payload, details } : r.data,
|
||||
createdAt: r.created_at,
|
||||
entry: entryObj,
|
||||
}
|
||||
@@ -579,6 +665,13 @@ export class AgentService {
|
||||
agent: {
|
||||
teamId,
|
||||
},
|
||||
OR: [
|
||||
{ type: { not: 'comment' } },
|
||||
{
|
||||
type: 'comment',
|
||||
userCommentId: { not: null },
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
return await paginateQuery(
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { migrateLegacyAgentAvatars } from './migration'
|
||||
import { getPresetAvatarBuffer } from './presets'
|
||||
|
||||
describe('migrateLegacyAgentAvatars', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
it('returns 0 when no legacy agents exist', async () => {
|
||||
const result = await migrateLegacyAgentAvatars(prisma)
|
||||
expect(result).toEqual({ migrated: 0, errors: 0 })
|
||||
})
|
||||
|
||||
it('migrates legacy agent base64 avatars to S3', async () => {
|
||||
const presetBase64 =
|
||||
'data:image/webp;base64,' + getPresetAvatarBuffer('avatar-1')!.toString('base64')
|
||||
const customBase64 =
|
||||
'data:image/webp;base64,UklGRtInAABXRUJQVlA4IMYnAACwmACdASosASwBPm0ylUekIyIhp1L7UIANiWVu3P6nGjxxkZZTIvmXwJRUeg/Jf+ndPZ0L4H6mVI3YJ/D/tH5j9vT9C/6f3AP1d/Xz+79l/zDfsj6yP+59R39j/y/sAf3r/VdZ36EXly+0Z+6/peaqTMC4q/u/7z45+fr5VuB3l+xvUg7u/5f+G9tH9D3q/Lv/a9Qj25/vvGH2lFyP2J9gX3F+8f9H1AfzfM/+G/0nsBfrb/0PZDvp/yf/H9gP+nf4j9lPeL/2PKX+yf8H2F/6D1Of3o9pv9uzXj2UCILxSqTJtnLrjF0IFB27Vo5QPaRJKEf73CFDiYb46kQh9ZimTAfOlbMytbJDx1VHoVO115vAHuMthHxJoOL/D/OOWk7DrZTEtbvTiXzvmXoZSFox8u7pVGZ4TmovMKAMAZtbdgS8p6MNEYUl2x4pPMe0bPqCNISczDauiMeOe/Bgos4oXilTgIe/4FB/KBRUIV0FiVRnmPndzO7QLVRnyoYI+oKPA/heN/n2wfElNcnOExdS6iaO/IIjkYWljnuIe8akKIWAkomDcGMr2fPIiNFXCNFxgRCTPFdGc+T/b2dMMLNdN3KjGjeGMzQe4UURLfl5oiFEw5Pu5y0fFhDnYhvS7Ezpsb54rvwP+ycBblMc5VRkMxpWgi7zlkUUOIzcsp8rJvOWHryMg36mIXa1xMNyp2rhlY3Vgul9+8VfT412zRHaz12VW/qh9l67/CDtCsnUBGO/e8J1mT9RAmfZTCKX+ecvAjhxyIeUEAg5olYzZSkgLNrAf+HFBHtihqoNZNUgRB+NKEfWCXPqAmsILE+/Ap/wD2FlHj0Q1Qk8PrAN9C/fyX/qgkJyQGu9cAmoYwicMlikN4oRxsA6WckScktHrWfdZYT6WV2xwpPXzy/IJL7TEXpY+W9Lwn/I3P8+q9N+d895Qu4TZMfUfou353e8Wn2UzcHQsKMI7fIZiW7QHhpYTgN41QTiY3CdoGpCOjBJ7cGZrYCpm3XGz2+FBkc1si2VAiuYI/4iWRjeO/MM28I+Qr93kQ+fpnyeMMsvk845A7+37z9M5b/IfOo2z+AywfCalW3nlFXyjVqphG4zq1Xe+FQmMjC8S8JNRMUDEfQnkEcCL/+cd6RkDsj4HAwG6VD3D23XacpKJf4/D1B/f+0v/cJrVvKEknKLNw1nz2gX7HKcAWB0Z+QZDp7Y2FkMAG17AnNVYSAqW7/xgalH6qTLexWRu6LYJVIdLLlriLzZOZlqvIXvawsT1GexNoi/OPUEgSlmUbNHEFe+NtoSuKzV0DgE0k3NXU8Fjjc/oeJL0J+XplQFqXhYg+r4yDOGN7h7xxyUm7IeUiDRLK9MkyQ7b+tRIUVg6pxhKRwj0r34SPFDlMWxhsfO/fYmq/5FbsACkYPysdQAjJzNqbbmX5A7OyZ0Y85gjBVDNHWMZswj7HJVofro0VpmGWUdzLSs/aBF0o2xNxcUdxqeqUFURISYHaX8Jx0ONcwIGQH3EperfP7ZiOxEKrot1BjTcWvYQ4LD4XkTAa+16dp5x4svp1Pp8/U5IGXD+WZAhpxIA+JaP16povKRq9O4hiZQyExaNml6lrO0G/olTvRSc9nq/PybXAw5uYhRIvclmBM27AAA='
|
||||
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Legacy Agent Preset',
|
||||
email: `legacy-preset-${Date.now()}@shumai.ai`,
|
||||
type: 'agent',
|
||||
image: presetBase64,
|
||||
},
|
||||
})
|
||||
|
||||
const customAgentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Legacy Agent Custom',
|
||||
email: `legacy-custom-${Date.now()}@shumai.ai`,
|
||||
type: 'agent',
|
||||
image: customBase64,
|
||||
},
|
||||
})
|
||||
|
||||
const humanUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Human User',
|
||||
email: `human-${Date.now()}@shumai.ai`,
|
||||
type: 'human',
|
||||
image: customBase64,
|
||||
},
|
||||
})
|
||||
|
||||
const modernAgentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Modern Agent',
|
||||
email: `modern-${Date.now()}@shumai.ai`,
|
||||
type: 'agent',
|
||||
image: 'files/existing-key.webp',
|
||||
},
|
||||
})
|
||||
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team' },
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'p',
|
||||
model: 'm',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: customAgentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'p',
|
||||
model: 'm',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const result = await migrateLegacyAgentAvatars(prisma)
|
||||
expect(result).toEqual({ migrated: 2, errors: 0 })
|
||||
|
||||
const updatedAgent = await prisma.user.findUnique({
|
||||
where: { id: agentUser.id },
|
||||
})
|
||||
expect(updatedAgent?.image).toMatch(/^files\/[A-Z0-9]{26}\.webp$/)
|
||||
|
||||
const updatedAgentRecord = await prisma.agent.findUnique({
|
||||
where: { id: agentUser.id },
|
||||
})
|
||||
expect((updatedAgentRecord?.config as unknown as PrismaJson.AgentConfig)?.avatarPreset).toBe(
|
||||
'avatar-1',
|
||||
)
|
||||
|
||||
const updatedCustomAgent = await prisma.user.findUnique({
|
||||
where: { id: customAgentUser.id },
|
||||
})
|
||||
expect(updatedCustomAgent?.image).toMatch(/^files\/[A-Z0-9]{26}\.webp$/)
|
||||
|
||||
const updatedCustomAgentRecord = await prisma.agent.findUnique({
|
||||
where: { id: customAgentUser.id },
|
||||
})
|
||||
expect(
|
||||
(updatedCustomAgentRecord?.config as unknown as PrismaJson.AgentConfig)?.avatarPreset,
|
||||
).toBeUndefined()
|
||||
|
||||
const updatedHuman = await prisma.user.findUnique({
|
||||
where: { id: humanUser.id },
|
||||
})
|
||||
expect(updatedHuman?.image).toBe(customBase64)
|
||||
|
||||
const updatedModern = await prisma.user.findUnique({
|
||||
where: { id: modernAgentUser.id },
|
||||
})
|
||||
expect(updatedModern?.image).toBe('files/existing-key.webp')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,101 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { ulid } from 'ulid'
|
||||
import { PRESET_AVATAR_IDS, getPresetAvatarBuffer } from './presets'
|
||||
|
||||
export async function migrateLegacyAgentAvatars(
|
||||
client: typeof prisma = prisma,
|
||||
): Promise<{ migrated: number; errors: number }> {
|
||||
const legacyUsers = await client.user.findMany({
|
||||
where: {
|
||||
type: 'agent',
|
||||
image: {
|
||||
startsWith: 'data:',
|
||||
},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
image: true,
|
||||
},
|
||||
})
|
||||
|
||||
if (legacyUsers.length === 0) {
|
||||
return { migrated: 0, errors: 0 }
|
||||
}
|
||||
|
||||
logger.info({ count: legacyUsers.length }, 'Starting migration of legacy agent avatars to S3')
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
let migrated = 0
|
||||
let errors = 0
|
||||
|
||||
for (const user of legacyUsers) {
|
||||
try {
|
||||
if (!user.image) continue
|
||||
|
||||
const match = user.image.match(/^data:([^;]+);base64,(.+)$/)
|
||||
let buffer: Buffer
|
||||
let contentType = 'image/webp'
|
||||
let ext = 'webp'
|
||||
|
||||
if (match) {
|
||||
contentType = match[1]
|
||||
buffer = Buffer.from(match[2], 'base64')
|
||||
const subtype = contentType.split('/')[1]
|
||||
if (subtype) ext = subtype
|
||||
} else {
|
||||
const commaIdx = user.image.indexOf(',')
|
||||
const rawBase64 = commaIdx !== -1 ? user.image.slice(commaIdx + 1) : user.image
|
||||
buffer = Buffer.from(rawBase64, 'base64')
|
||||
}
|
||||
|
||||
let matchedPresetId: string | undefined
|
||||
for (const id of PRESET_AVATAR_IDS) {
|
||||
const presetBuf = getPresetAvatarBuffer(id)
|
||||
if (presetBuf && buffer.equals(presetBuf)) {
|
||||
matchedPresetId = id
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const key = `files/${ulid()}.${ext}`
|
||||
await s3Service.putObject(bucket, key, buffer, buffer.length, contentType)
|
||||
|
||||
await client.user.update({
|
||||
where: { id: user.id },
|
||||
data: { image: key },
|
||||
})
|
||||
|
||||
if (matchedPresetId) {
|
||||
const agent = await client.agent.findUnique({
|
||||
where: { id: user.id },
|
||||
select: { config: true },
|
||||
})
|
||||
if (agent) {
|
||||
const currentConfig = (agent.config as unknown as PrismaJson.AgentConfig) || {}
|
||||
await client.agent.update({
|
||||
where: { id: user.id },
|
||||
data: {
|
||||
config: {
|
||||
...currentConfig,
|
||||
avatarPreset: matchedPresetId,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
migrated++
|
||||
logger.info(
|
||||
{ userId: user.id, key, avatarPreset: matchedPresetId },
|
||||
'Successfully migrated agent avatar to S3',
|
||||
)
|
||||
} catch (err: unknown) {
|
||||
errors++
|
||||
logger.error({ userId: user.id, err }, 'Failed to migrate legacy agent avatar')
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ migrated, errors }, 'Completed migration of legacy agent avatars')
|
||||
return { migrated, errors }
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
getPresetAvatarBuffer,
|
||||
isPresetAvatarId,
|
||||
PRESET_AVATAR_IDS,
|
||||
type PresetAvatarId,
|
||||
} from './presets'
|
||||
|
||||
describe('agent presets', () => {
|
||||
it('identifies preset avatar IDs correctly', () => {
|
||||
expect(PRESET_AVATAR_IDS).toHaveLength(8)
|
||||
for (const id of PRESET_AVATAR_IDS) {
|
||||
expect(isPresetAvatarId(id)).toBe(true)
|
||||
}
|
||||
expect(isPresetAvatarId('avatar-999')).toBe(false)
|
||||
expect(isPresetAvatarId('')).toBe(false)
|
||||
expect(isPresetAvatarId(null)).toBe(false)
|
||||
expect(isPresetAvatarId(undefined)).toBe(false)
|
||||
})
|
||||
|
||||
it('returns valid buffer for all preset avatars', () => {
|
||||
for (const id of PRESET_AVATAR_IDS) {
|
||||
const buf = getPresetAvatarBuffer(id)
|
||||
expect(buf).toBeInstanceOf(Buffer)
|
||||
expect(buf?.length).toBeGreaterThan(5000)
|
||||
}
|
||||
expect(getPresetAvatarBuffer('unknown' as PresetAvatarId)).toBeNull()
|
||||
})
|
||||
|
||||
it('provides AVAILABLE_AVATARS with 8 previews via dtos', async () => {
|
||||
const { AVAILABLE_AVATARS } = await import('@shumai/dtos')
|
||||
expect(AVAILABLE_AVATARS).toHaveLength(8)
|
||||
for (const avatar of AVAILABLE_AVATARS) {
|
||||
expect(avatar.id).toMatch(/^avatar-[1-8]$/)
|
||||
expect(avatar.preview).toMatch(/^data:image\/webp;base64,/)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,22 @@
|
||||
import {
|
||||
PRESET_AVATAR_IDS,
|
||||
type PresetAvatarId,
|
||||
isPresetAvatarId,
|
||||
PRESET_AVATAR_BASE64,
|
||||
} from '@shumai/dtos'
|
||||
|
||||
export { PRESET_AVATAR_IDS, type PresetAvatarId, isPresetAvatarId }
|
||||
|
||||
const bufferCache = new Map<PresetAvatarId, Buffer>()
|
||||
|
||||
export function getPresetAvatarBuffer(id: PresetAvatarId): Buffer | null {
|
||||
const cached = bufferCache.get(id)
|
||||
if (cached) return cached
|
||||
|
||||
const b64 = PRESET_AVATAR_BASE64[id]
|
||||
if (!b64) return null
|
||||
|
||||
const buf = Buffer.from(b64, 'base64')
|
||||
bufferCache.set(id, buf)
|
||||
return buf
|
||||
}
|
||||
@@ -626,6 +626,39 @@ describe('AssetService', () => {
|
||||
).rejects.toThrow('Can only move one file at a time into a version stack')
|
||||
})
|
||||
|
||||
it('rejects reparenting a folder into itself', async () => {
|
||||
const { user, assets } = await setupBasicAssets()
|
||||
await expect(
|
||||
assetService.reparentAssets({
|
||||
assetIds: [assets.folderA.id],
|
||||
newParentId: assets.folderA.id,
|
||||
creatorId: user.id,
|
||||
}),
|
||||
).rejects.toThrow('Cannot move an asset into itself or its own descendant')
|
||||
})
|
||||
|
||||
it('rejects reparenting a folder into its own descendant', async () => {
|
||||
const { user, assets, project } = await setupBasicAssets()
|
||||
const subfolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'subfolderA',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
parentId: assets.folderA.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(
|
||||
assetService.reparentAssets({
|
||||
assetIds: [assets.folderA.id],
|
||||
newParentId: subfolder.id,
|
||||
creatorId: user.id,
|
||||
}),
|
||||
).rejects.toThrow('Cannot move an asset into itself or its own descendant')
|
||||
})
|
||||
|
||||
describe('fileCount verification', () => {
|
||||
it('createAsset should increment parent fileCount', async () => {
|
||||
const { user, assets, project } = await setupBasicAssets()
|
||||
@@ -3821,6 +3854,65 @@ describe('AssetService — natural sort by name', () => {
|
||||
expect(triggerSpy).toHaveBeenCalledWith(expect.anything(), fileInfo.id, team.id, project.id)
|
||||
})
|
||||
|
||||
it('creates a file with agentId and returns agent info in toAssetInfos', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'HumanUser', email: `human-${Date.now()}@example.com` },
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 'AgentFileTeam' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'AgentFileProject', teamId: team.id },
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'RootFolder',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'AssetBot',
|
||||
email: `bot-${Date.now()}@example.com`,
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'test', model: 'test' },
|
||||
},
|
||||
})
|
||||
|
||||
vi.spyOn(uploadService, 'triggerPostUploadWorkflows').mockResolvedValue(undefined)
|
||||
|
||||
const fileInfo = await assetService.createFile({
|
||||
parentId: rootFolder.id,
|
||||
name: 'generated.png',
|
||||
key: 'files/test/generated.png',
|
||||
sizeByte: 2048,
|
||||
contentType: 'image/png',
|
||||
creatorId: user.id,
|
||||
agentId: agent.id,
|
||||
})
|
||||
|
||||
expect(fileInfo.id).toBeDefined()
|
||||
expect(fileInfo.agentId).toBe(agent.id)
|
||||
expect(fileInfo.agent).toEqual({
|
||||
id: agent.id,
|
||||
name: 'AssetBot',
|
||||
})
|
||||
|
||||
const fetched = await assetService.getAsset({ assetId: fileInfo.id })
|
||||
expect(fetched.agentId).toBe(agent.id)
|
||||
expect(fetched.agent).toEqual({
|
||||
id: agent.id,
|
||||
name: 'AssetBot',
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects if metadata contains invalid keys', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'FileUser2', email: `fileuser2-${Date.now()}@example.com` },
|
||||
@@ -4085,6 +4177,83 @@ describe('AssetService — natural sort by name', () => {
|
||||
expect(rootFolderAfter?.fileCount).toBe(1)
|
||||
expect(rootFolderAfter?.sizeByte).toBe(2000n)
|
||||
})
|
||||
|
||||
it('creates a version with agentId and preserves agent provenance in version stack', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'VersionUser2', email: `veruser2-${Date.now()}@example.com` },
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 'VersionTeam2' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'VersionProject2', teamId: team.id },
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'RootFolder',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'VersionBot',
|
||||
email: `verbot-${Date.now()}@example.com`,
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'test', model: 'test' },
|
||||
},
|
||||
})
|
||||
|
||||
const v1 = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'document.txt',
|
||||
type: AssetType.file,
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
status: AssetStatus.uploaded,
|
||||
sizeByte: 100n,
|
||||
creatorId: user.id,
|
||||
},
|
||||
})
|
||||
|
||||
vi.spyOn(uploadService, 'triggerPostUploadWorkflows').mockResolvedValue(undefined)
|
||||
|
||||
const v2Info = await assetService.createVersion({
|
||||
parentId: v1.id,
|
||||
name: 'document.txt',
|
||||
key: 'files/test/v2.txt',
|
||||
sizeByte: 200,
|
||||
contentType: 'text/plain',
|
||||
creatorId: user.id,
|
||||
agentId: agent.id,
|
||||
})
|
||||
|
||||
expect(v2Info.id).toBeDefined()
|
||||
expect(v2Info.agentId).toBe(agent.id)
|
||||
expect(v2Info.agent).toEqual({
|
||||
id: agent.id,
|
||||
name: 'VersionBot',
|
||||
})
|
||||
|
||||
const stack = await prisma.asset.findFirst({
|
||||
where: { parentId: rootFolder.id, type: AssetType.version_stack },
|
||||
})
|
||||
expect(stack).toBeDefined()
|
||||
|
||||
const stackVersions = await assetService.getStackVersions(stack!.id)
|
||||
expect(stackVersions).toHaveLength(2)
|
||||
const v2StackVersion = stackVersions.find((v) => v.id === v2Info.id)
|
||||
expect(v2StackVersion?.agent).toEqual({
|
||||
id: agent.id,
|
||||
name: 'VersionBot',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('listChildren type filtering', () => {
|
||||
|
||||
@@ -34,20 +34,38 @@ import { uploadService } from '@shumai/core/src/upload/upload'
|
||||
|
||||
export const assetInclude = {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
where: { isDeleted: false },
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
},
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
where: { isDeleted: false },
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
},
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
} as const
|
||||
|
||||
@@ -101,30 +119,7 @@ export class AssetService {
|
||||
|
||||
const assets = await this.prismaClient.asset.findMany({
|
||||
where: { id: { in: ids } },
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
include: assetInclude,
|
||||
})
|
||||
|
||||
const assetMap = new Map(assets.map((a) => [a.id, a]))
|
||||
@@ -228,6 +223,20 @@ export class AssetService {
|
||||
|
||||
if (assetsToMove.length === 0) return
|
||||
|
||||
// Check if newParentId is equal to or a descendant of any asset being moved
|
||||
const ancestors = await tx.$queryRaw<{ id: string }[]>`
|
||||
WITH RECURSIVE ancestor AS (
|
||||
SELECT id, parent_id FROM assets WHERE id = ${newParent.id}
|
||||
UNION ALL
|
||||
SELECT a.id, a.parent_id FROM assets a
|
||||
INNER JOIN ancestor d ON a.id = d.parent_id
|
||||
)
|
||||
SELECT id FROM ancestor WHERE id IN (${Prisma.join(req.assetIds)}) LIMIT 1;
|
||||
`
|
||||
if (ancestors.length > 0) {
|
||||
throw new Error('Cannot move an asset into itself or its own descendant')
|
||||
}
|
||||
|
||||
if (newParent.type === AssetType.version_stack) {
|
||||
for (const a of assetsToMove) {
|
||||
if (a.type !== AssetType.file) {
|
||||
@@ -711,30 +720,7 @@ export class AssetService {
|
||||
async (skip, take) => {
|
||||
return this.prismaClient.asset.findMany({
|
||||
where,
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
include: assetInclude,
|
||||
orderBy,
|
||||
skip,
|
||||
take,
|
||||
@@ -798,22 +784,30 @@ export class AssetService {
|
||||
if (req.sizeByte) data.sizeByte = req.sizeByte
|
||||
if (req.contentType) data.mediaType = req.contentType
|
||||
if (req.creatorId) data.creator = { connect: { id: req.creatorId } }
|
||||
if (req.agentId) data.agent = { connect: { id: req.agentId } }
|
||||
|
||||
const executeInTx = async (tx: Prisma.TransactionClient) => {
|
||||
const createdAsset = await tx.asset.create({
|
||||
data,
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
},
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
@@ -821,7 +815,12 @@ export class AssetService {
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
},
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
@@ -853,9 +852,10 @@ export class AssetService {
|
||||
sizeByte: number
|
||||
contentType: string
|
||||
creatorId: string
|
||||
agentId?: string
|
||||
metadata?: Record<string, unknown>
|
||||
}): Promise<AssetInfo> {
|
||||
const { parentId, name, key, sizeByte, contentType, creatorId, metadata } = params
|
||||
const { parentId, name, key, sizeByte, contentType, creatorId, agentId, metadata } = params
|
||||
|
||||
const parentAsset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: parentId },
|
||||
@@ -884,6 +884,7 @@ export class AssetService {
|
||||
sizeByte,
|
||||
contentType,
|
||||
creatorId,
|
||||
agentId,
|
||||
},
|
||||
tx,
|
||||
)
|
||||
@@ -916,9 +917,10 @@ export class AssetService {
|
||||
sizeByte: number
|
||||
contentType: string
|
||||
creatorId: string
|
||||
agentId?: string
|
||||
metadata?: Record<string, unknown>
|
||||
}): Promise<AssetInfo> {
|
||||
const { parentId, name, key, sizeByte, contentType, creatorId, metadata } = params
|
||||
const { parentId, name, key, sizeByte, contentType, creatorId, agentId, metadata } = params
|
||||
|
||||
const targetAsset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: parentId },
|
||||
@@ -961,6 +963,7 @@ export class AssetService {
|
||||
sizeByte,
|
||||
contentType,
|
||||
creatorId,
|
||||
agentId,
|
||||
},
|
||||
tx,
|
||||
)
|
||||
@@ -1006,30 +1009,7 @@ export class AssetService {
|
||||
async getAsset(req: GetAssetRequest): Promise<AssetInfo> {
|
||||
const a = await this.prismaClient.asset.findUnique({
|
||||
where: { id: req.assetId },
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
target: {
|
||||
include: {
|
||||
creator: true,
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
children: {
|
||||
where: { isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
take: 3,
|
||||
orderBy: { sortIndex: 'asc' },
|
||||
},
|
||||
},
|
||||
include: assetInclude,
|
||||
})
|
||||
if (!a) throw new Error('Asset not found')
|
||||
|
||||
@@ -1914,7 +1894,12 @@ export class AssetService {
|
||||
if (stackIds.size > 0) {
|
||||
const allVersions = await this.prismaClient.asset.findMany({
|
||||
where: { parentId: { in: Array.from(stackIds) }, isDeleted: false },
|
||||
include: { creator: true, metadataValues: true, storageKey: true },
|
||||
include: {
|
||||
creator: true,
|
||||
agent: { include: { user: true } },
|
||||
metadataValues: true,
|
||||
storageKey: true,
|
||||
},
|
||||
})
|
||||
for (const v of allVersions) {
|
||||
const list = versionsMap.get(v.parentId!) || []
|
||||
@@ -1974,6 +1959,12 @@ export class AssetService {
|
||||
image: await getAvatarUrl(v.creator.image),
|
||||
}
|
||||
: null,
|
||||
agent: v.agent
|
||||
? {
|
||||
id: v.agent.id,
|
||||
name: v.agent.user.name,
|
||||
}
|
||||
: null,
|
||||
}
|
||||
}),
|
||||
),
|
||||
@@ -2026,6 +2017,13 @@ export class AssetService {
|
||||
}
|
||||
: null
|
||||
|
||||
const agent = latestVersion.agent
|
||||
? {
|
||||
id: latestVersion.agent.id,
|
||||
name: latestVersion.agent.user.name,
|
||||
}
|
||||
: null
|
||||
|
||||
const fieldValues: FieldValueInfo[] = []
|
||||
if ('metadataValues' in latestVersion && latestVersion.metadataValues) {
|
||||
for (const mv of latestVersion.metadataValues) {
|
||||
@@ -2101,6 +2099,8 @@ export class AssetService {
|
||||
deletedAt: a.deletedAt ? a.deletedAt.toISOString() : null,
|
||||
projectId: a.projectId,
|
||||
creator,
|
||||
agentId: latestVersion.agentId ?? null,
|
||||
agent,
|
||||
fieldValues,
|
||||
sortIndex: a.sortIndex,
|
||||
hasAgentsMd: hasAgentMdSet.has(a.id),
|
||||
@@ -2119,11 +2119,12 @@ export class AssetService {
|
||||
previewUrl: string | null
|
||||
createdAt: string
|
||||
creator: { id: string; name: string | null; image?: string | null } | null
|
||||
agent?: { id: string; name: string } | null
|
||||
}>
|
||||
> {
|
||||
const versions = await this.prismaClient.asset.findMany({
|
||||
where: { parentId: stackId, isDeleted: false },
|
||||
include: { creator: true },
|
||||
include: { creator: true, agent: { include: { user: true } } },
|
||||
})
|
||||
|
||||
versions.sort((x, y) => {
|
||||
@@ -2147,6 +2148,12 @@ export class AssetService {
|
||||
image: await getAvatarUrl(v.creator.image),
|
||||
}
|
||||
: null,
|
||||
agent: v.agent
|
||||
? {
|
||||
id: v.agent.id,
|
||||
name: v.agent.user.name,
|
||||
}
|
||||
: null,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -116,4 +116,57 @@ describe('AuditLogService', () => {
|
||||
expect(result.nodes.map((n) => n.action)).toContain(AuditAction.kanban_task_create)
|
||||
expect(result.nodes.map((n) => n.action)).toContain(AuditAction.kanban_goal_create)
|
||||
})
|
||||
|
||||
it('records and filters agent provenance in audit logs', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Audit Human User', email: 'audit_human@example.com', password: 'password' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Agent Audit Team' },
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: {
|
||||
name: 'Creation Bot',
|
||||
email: 'creation_bot@example.com',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'test', model: 'test' },
|
||||
},
|
||||
})
|
||||
|
||||
const log = await auditLogService.logAction({
|
||||
action: AuditAction.file_create,
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
agentId: agent.id,
|
||||
itemId: 'file-xyz',
|
||||
})
|
||||
|
||||
expect(log).toBeDefined()
|
||||
expect(log?.agentId).toBe(agent.id)
|
||||
|
||||
const resultWithAgent = await auditLogService.listAuditLogs({
|
||||
teamId: team.id,
|
||||
agentIds: [agent.id],
|
||||
})
|
||||
|
||||
expect(resultWithAgent.total).toBe(1)
|
||||
expect(resultWithAgent.nodes[0].agentId).toBe(agent.id)
|
||||
expect(resultWithAgent.nodes[0].agent).toEqual({
|
||||
id: agent.id,
|
||||
name: 'Creation Bot',
|
||||
})
|
||||
|
||||
const resultOtherAgent = await auditLogService.listAuditLogs({
|
||||
teamId: team.id,
|
||||
agentIds: ['non-existent-agent'],
|
||||
})
|
||||
expect(resultOtherAgent.total).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface LogActionParams {
|
||||
action: AuditAction
|
||||
teamId: string
|
||||
userId?: string
|
||||
agentId?: string
|
||||
projectId?: string
|
||||
itemId?: string
|
||||
}
|
||||
@@ -16,6 +17,7 @@ export interface ListAuditLogsParams {
|
||||
teamId: string
|
||||
actions?: AuditAction[]
|
||||
userIds?: string[]
|
||||
agentIds?: string[]
|
||||
itemId?: string
|
||||
first?: number
|
||||
after?: string
|
||||
@@ -29,6 +31,7 @@ export class AuditLogService {
|
||||
action: params.action,
|
||||
teamId: params.teamId,
|
||||
userId: params.userId,
|
||||
agentId: params.agentId,
|
||||
projectId: params.projectId,
|
||||
itemId: params.itemId,
|
||||
},
|
||||
@@ -52,6 +55,10 @@ export class AuditLogService {
|
||||
where.userId = { in: params.userIds }
|
||||
}
|
||||
|
||||
if (params.agentIds && params.agentIds.length > 0) {
|
||||
where.agentId = { in: params.agentIds }
|
||||
}
|
||||
|
||||
if (params.itemId && params.itemId.trim() !== '') {
|
||||
where.itemId = { contains: params.itemId.trim(), mode: 'insensitive' }
|
||||
}
|
||||
@@ -60,6 +67,7 @@ export class AuditLogService {
|
||||
(skip, take) =>
|
||||
prisma.auditLog.findMany({
|
||||
where,
|
||||
include: { agent: { include: { user: true } } },
|
||||
orderBy: { id: 'desc' },
|
||||
skip,
|
||||
take,
|
||||
@@ -74,6 +82,8 @@ export class AuditLogService {
|
||||
action: item.action,
|
||||
teamId: item.teamId,
|
||||
userId: item.userId,
|
||||
agentId: item.agentId,
|
||||
agent: item.agent ? { id: item.agent.id, name: item.agent.user.name } : null,
|
||||
projectId: item.projectId,
|
||||
itemId: item.itemId,
|
||||
createdAt: item.createdAt.toISOString(),
|
||||
|
||||
@@ -988,5 +988,29 @@ describe('ChatService', () => {
|
||||
const msg2 = mapEntryToMessage(emptyEntry)
|
||||
expect(msg2).toBeNull()
|
||||
})
|
||||
|
||||
it('should forward annotations and second to workflow task payload', async () => {
|
||||
const { user, team, project } = await setupBasicData()
|
||||
|
||||
const { sessionId, taskId } = await chatService.startOrContinueChat(user, team.id, {
|
||||
agentId: 'test-agent-id',
|
||||
textPrompt: 'Check out this area',
|
||||
projectId: project.id,
|
||||
annotations: [{ type: 'rectangle', x: 10, y: 20, width: 30, height: 40 }],
|
||||
second: 8.5,
|
||||
})
|
||||
|
||||
expect(sessionId).toBeDefined()
|
||||
expect(taskId).toBeDefined()
|
||||
|
||||
const task = await prisma.workflowTask.findUnique({
|
||||
where: { id: taskId },
|
||||
})
|
||||
expect(task).toBeDefined()
|
||||
expect(task?.payload?.agent?.second).toBe(8.5)
|
||||
expect(task?.payload?.agent?.annotations).toEqual([
|
||||
{ type: 'rectangle', x: 10, y: 20, width: 30, height: 40 },
|
||||
])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -199,6 +199,8 @@ export class ChatService {
|
||||
sessionId: passedSessionId,
|
||||
contextAssetId,
|
||||
projectId: passedProjectId,
|
||||
second,
|
||||
annotations,
|
||||
} = req
|
||||
|
||||
// 1. Permissions verification
|
||||
@@ -401,6 +403,8 @@ export class ChatService {
|
||||
userId: user.id,
|
||||
isNewChat: !passedSessionId,
|
||||
hasAssetChanged,
|
||||
second: second ?? undefined,
|
||||
annotations: annotations ?? undefined,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -28,3 +28,4 @@ export * from './gotenberg/gotenberg'
|
||||
export * from './mcp/mcp-service'
|
||||
export * from './quota/quota-service'
|
||||
export * from './kanban/kanban'
|
||||
export * from './media-generation/media-generation'
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { mediaGenerationService, BUILTIN_MEDIA_PROVIDERS } from './media-generation'
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
|
||||
import * as aiModule from 'ai'
|
||||
|
||||
vi.mock('ai', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('ai')>()
|
||||
return {
|
||||
...actual,
|
||||
generateText: vi.fn(),
|
||||
}
|
||||
})
|
||||
|
||||
describe('MediaGenerationService', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
let teamId: string
|
||||
const originalEnv = process.env
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks()
|
||||
process.env = { ...originalEnv }
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Media Team',
|
||||
settings: {
|
||||
transcode: { videoStrategy: 'best_match' },
|
||||
},
|
||||
},
|
||||
})
|
||||
teamId = team.id
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
vi.clearAllMocks()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('should list all 18 providers in getSettings', async () => {
|
||||
const settings = await mediaGenerationService.getSettings(teamId)
|
||||
expect(settings.providers).toHaveLength(18)
|
||||
expect(settings.enabledModels).toEqual([])
|
||||
|
||||
const openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
expect(openai).toBeDefined()
|
||||
expect(openai?.defaultEnvKey).toBe('OPENAI_API_KEY')
|
||||
expect(openai?.supportedTypes).toEqual(['image'])
|
||||
})
|
||||
|
||||
it('should detect API key from environment variable', async () => {
|
||||
process.env.OPENAI_API_KEY = 'sk-test-env-key'
|
||||
const settings = await mediaGenerationService.getSettings(teamId)
|
||||
const openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
|
||||
expect(openai?.apiKeyConfigured).toBe(true)
|
||||
expect(openai?.status).toBe('configured_env')
|
||||
expect(openai?.customApiKeyOrEnv).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should allow setting custom API key and custom env variable', async () => {
|
||||
// 1. Set literal API key
|
||||
await mediaGenerationService.updateProviderApiKey(teamId, 'openai', 'sk-literal-key')
|
||||
let settings = await mediaGenerationService.getSettings(teamId)
|
||||
let openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
|
||||
expect(openai?.apiKeyConfigured).toBe(true)
|
||||
expect(openai?.status).toBe('configured_custom')
|
||||
expect(openai?.customApiKeyOrEnv).toBe('sk-literal-key')
|
||||
|
||||
// 2. Set custom ENV var name
|
||||
process.env.MY_CUSTOM_OPENAI_ENV = 'sk-resolved-custom-env'
|
||||
await mediaGenerationService.updateProviderApiKey(teamId, 'openai', 'MY_CUSTOM_OPENAI_ENV')
|
||||
settings = await mediaGenerationService.getSettings(teamId)
|
||||
openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
|
||||
expect(openai?.apiKeyConfigured).toBe(true)
|
||||
expect(openai?.status).toBe('configured_custom')
|
||||
expect(openai?.customApiKeyOrEnv).toBe('MY_CUSTOM_OPENAI_ENV')
|
||||
expect(mediaGenerationService.resolveApiKey('openai', 'MY_CUSTOM_OPENAI_ENV')).toBe(
|
||||
'sk-resolved-custom-env',
|
||||
)
|
||||
|
||||
// 3. Clear custom key
|
||||
await mediaGenerationService.updateProviderApiKey(teamId, 'openai', '')
|
||||
delete process.env.OPENAI_API_KEY
|
||||
settings = await mediaGenerationService.getSettings(teamId)
|
||||
openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
|
||||
expect(openai?.apiKeyConfigured).toBe(false)
|
||||
expect(openai?.status).toBe('not_configured')
|
||||
expect(openai?.customApiKeyOrEnv).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should update provider apiKey and models atomically with updateProvider', async () => {
|
||||
// Enable a model for fal first
|
||||
await mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'image',
|
||||
provider: 'fal',
|
||||
modelId: 'fal-ai/flux/dev',
|
||||
})
|
||||
|
||||
// Update openai apiKey and models
|
||||
await mediaGenerationService.updateProvider(teamId, 'openai', {
|
||||
apiKey: 'sk-test-atomic',
|
||||
models: [
|
||||
{ type: 'image', modelId: 'dall-e-3', name: 'DALL-E 3' },
|
||||
{ type: 'image', modelId: 'gpt-image-2', name: 'GPT Image 2' },
|
||||
],
|
||||
})
|
||||
|
||||
let settings = await mediaGenerationService.getSettings(teamId)
|
||||
let openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
expect(openai?.apiKeyConfigured).toBe(true)
|
||||
expect(openai?.status).toBe('configured_custom')
|
||||
expect(openai?.customApiKeyOrEnv).toBe('sk-test-atomic')
|
||||
|
||||
// Enabled models should have the 2 openai models plus the 1 fal model
|
||||
expect(settings.enabledModels).toHaveLength(3)
|
||||
const openaiModels = settings.enabledModels.filter((m) => m.provider === 'openai')
|
||||
expect(openaiModels).toHaveLength(2)
|
||||
expect(openaiModels.map((m) => m.modelId)).toContain('dall-e-3')
|
||||
expect(openaiModels.map((m) => m.modelId)).toContain('gpt-image-2')
|
||||
|
||||
// Now update openai with only 1 model enabled (disable gpt-image-2)
|
||||
await mediaGenerationService.updateProvider(teamId, 'openai', {
|
||||
models: [{ type: 'image', modelId: 'dall-e-3', name: 'DALL-E 3' }],
|
||||
})
|
||||
|
||||
settings = await mediaGenerationService.getSettings(teamId)
|
||||
openai = settings.providers.find((p) => p.provider === 'openai')
|
||||
// apiKey should remain unchanged
|
||||
expect(openai?.customApiKeyOrEnv).toBe('sk-test-atomic')
|
||||
const nextOpenaiModels = settings.enabledModels.filter((m) => m.provider === 'openai')
|
||||
expect(nextOpenaiModels).toHaveLength(1)
|
||||
expect(nextOpenaiModels[0].modelId).toBe('dall-e-3')
|
||||
// Fal model is still there
|
||||
expect(settings.enabledModels.find((m) => m.provider === 'fal')).toBeDefined()
|
||||
})
|
||||
|
||||
it('should add enabled models and reject invalid configurations', async () => {
|
||||
// Valid model addition
|
||||
const added = await mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
name: 'DALL-E 3 High Quality',
|
||||
})
|
||||
|
||||
expect(added.id).toBeDefined()
|
||||
expect(added.type).toBe('image')
|
||||
expect(added.provider).toBe('openai')
|
||||
expect(added.modelId).toBe('dall-e-3')
|
||||
expect(added.name).toBe('DALL-E 3 High Quality')
|
||||
|
||||
const settings = await mediaGenerationService.getSettings(teamId)
|
||||
expect(settings.enabledModels).toHaveLength(1)
|
||||
expect(settings.enabledModels[0].id).toBe(added.id)
|
||||
|
||||
// Rejects duplicate model addition
|
||||
await expect(
|
||||
mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
}),
|
||||
).rejects.toThrow('already enabled')
|
||||
|
||||
// Rejects unsupported type for provider (OpenAI does not support video)
|
||||
await expect(
|
||||
mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'video',
|
||||
provider: 'openai',
|
||||
modelId: 'some-video-model',
|
||||
}),
|
||||
).rejects.toThrow('does not support video')
|
||||
|
||||
// Rejects unknown provider
|
||||
await expect(
|
||||
mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'image',
|
||||
provider: 'unknown-provider',
|
||||
modelId: 'model-1',
|
||||
}),
|
||||
).rejects.toThrow('Unknown media provider')
|
||||
})
|
||||
|
||||
it('should remove enabled models', async () => {
|
||||
const model = await mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'image',
|
||||
provider: 'fal',
|
||||
modelId: 'fal-ai/flux/dev',
|
||||
})
|
||||
|
||||
let settings = await mediaGenerationService.getSettings(teamId)
|
||||
expect(settings.enabledModels).toHaveLength(1)
|
||||
|
||||
await mediaGenerationService.removeEnabledModel(teamId, model.id)
|
||||
settings = await mediaGenerationService.getSettings(teamId)
|
||||
expect(settings.enabledModels).toHaveLength(0)
|
||||
|
||||
// Throws if model does not exist
|
||||
await expect(mediaGenerationService.removeEnabledModel(teamId, 'non-existent')).rejects.toThrow(
|
||||
'not found',
|
||||
)
|
||||
})
|
||||
|
||||
it('should return valid models only when provider has active API key', async () => {
|
||||
// Add an image model for OpenAI and a video model for Google
|
||||
await mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'image',
|
||||
provider: 'openai',
|
||||
modelId: 'dall-e-3',
|
||||
})
|
||||
await mediaGenerationService.addEnabledModel(teamId, {
|
||||
type: 'video',
|
||||
provider: 'google',
|
||||
modelId: 'veo-2.0-generate-001',
|
||||
})
|
||||
|
||||
// Neither key is configured yet
|
||||
delete process.env.OPENAI_API_KEY
|
||||
delete process.env.GEMINI_API_KEY
|
||||
let valid = await mediaGenerationService.getValidModels(teamId)
|
||||
expect(valid.imageModels).toHaveLength(0)
|
||||
expect(valid.videoModels).toHaveLength(0)
|
||||
|
||||
// Configure OpenAI key
|
||||
process.env.OPENAI_API_KEY = 'sk-test'
|
||||
valid = await mediaGenerationService.getValidModels(teamId)
|
||||
expect(valid.imageModels).toHaveLength(1)
|
||||
expect(valid.imageModels[0].modelId).toBe('dall-e-3')
|
||||
expect(valid.videoModels).toHaveLength(0)
|
||||
|
||||
// Configure Google key
|
||||
process.env.GEMINI_API_KEY = 'gemini-key'
|
||||
valid = await mediaGenerationService.getValidModels(teamId)
|
||||
expect(valid.imageModels).toHaveLength(1)
|
||||
expect(valid.videoModels).toHaveLength(1)
|
||||
expect(valid.videoModels[0].modelId).toBe('veo-2.0-generate-001')
|
||||
})
|
||||
|
||||
it('should instantiate image and video models across all supported providers', () => {
|
||||
for (const [providerKey, def] of Object.entries(BUILTIN_MEDIA_PROVIDERS)) {
|
||||
if (def.supportedTypes.includes('image')) {
|
||||
const imageModel = def.curatedModels.find((m) => m.type === 'image')!
|
||||
const instance = mediaGenerationService.getImageModel(
|
||||
providerKey,
|
||||
imageModel.modelId,
|
||||
'mock-api-key',
|
||||
)
|
||||
expect(instance).toBeDefined()
|
||||
expect(instance.modelId).toBe(imageModel.modelId)
|
||||
}
|
||||
|
||||
if (def.supportedTypes.includes('video')) {
|
||||
const videoModel = def.curatedModels.find((m) => m.type === 'video')!
|
||||
const instance = mediaGenerationService.getVideoModel(
|
||||
providerKey,
|
||||
videoModel.modelId,
|
||||
'mock-api-key',
|
||||
)
|
||||
expect(instance).toBeDefined()
|
||||
expect(instance.modelId).toBe(videoModel.modelId)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('should include gemini-omni-1.1-flash as the default video model in Google curated models', () => {
|
||||
const googleVideoModels = mediaGenerationService.getCuratedModels('google', 'video')
|
||||
expect(googleVideoModels.length).toBeGreaterThan(0)
|
||||
expect(googleVideoModels[0].modelId).toBe('gemini-omni-1.1-flash')
|
||||
expect(googleVideoModels[0].name).toBe('Gemini Omni 1.1 Flash')
|
||||
expect(googleVideoModels[0].type).toBe('video')
|
||||
})
|
||||
|
||||
it('should generate video via interactions API for gemini-omni-1.1-flash in text_to_video mode', async () => {
|
||||
const mockGenerateText = vi.mocked(aiModule.generateText).mockResolvedValue({
|
||||
files: [
|
||||
{
|
||||
mediaType: 'video/mp4',
|
||||
uint8Array: new Uint8Array([10, 20, 30, 40]),
|
||||
base64: Buffer.from([10, 20, 30, 40]).toString('base64'),
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
} as unknown as Awaited<ReturnType<typeof aiModule.generateText>>)
|
||||
|
||||
const result = await mediaGenerationService.generateVideo({
|
||||
provider: 'google',
|
||||
modelId: 'gemini-omni-1.1-flash',
|
||||
apiKey: 'test-gemini-key',
|
||||
mode: 'text_to_video',
|
||||
prompt: 'A cinematic drone shot over rolling hills',
|
||||
aspectRatio: '16:9',
|
||||
resolution: '1280x720',
|
||||
duration: 5,
|
||||
fps: 24,
|
||||
generateAudio: true,
|
||||
seed: 42,
|
||||
})
|
||||
|
||||
expect(mockGenerateText).toHaveBeenCalledTimes(1)
|
||||
const callArgs = mockGenerateText.mock.calls[0][0]
|
||||
expect(callArgs.seed).toBe(42)
|
||||
expect(callArgs.system).toContain('Aspect ratio: 16:9')
|
||||
expect(callArgs.system).toContain('Resolution: 720p (1280x720)')
|
||||
expect(callArgs.system).toContain('Duration: 5 seconds')
|
||||
expect(callArgs.system).toContain('Frame rate: 24 fps')
|
||||
expect(callArgs.system).toContain('Audio: generate synchronized audio')
|
||||
const messages = callArgs.messages as Array<{
|
||||
role: string
|
||||
content: Array<{ type: string; text?: string }>
|
||||
}>
|
||||
expect(messages[0].content[0].text).toContain(
|
||||
'[Video parameters: Aspect ratio: 16:9, Resolution: 720p (1280x720), Duration: 5 seconds, Frame rate: 24 fps, Audio: generate synchronized audio]',
|
||||
)
|
||||
expect(callArgs.providerOptions).toEqual({
|
||||
google: {
|
||||
responseModalities: ['video'],
|
||||
store: false,
|
||||
},
|
||||
})
|
||||
expect(result.mimeType).toBe('video/mp4')
|
||||
expect(result.buffer).toEqual(Buffer.from([10, 20, 30, 40]))
|
||||
})
|
||||
|
||||
it('should generate video via interactions API for gemini-omni-1.1-flash in image_to_video mode', async () => {
|
||||
const fakeImageBuffer = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
||||
const mockGenerateText = vi.mocked(aiModule.generateText).mockResolvedValue({
|
||||
files: [
|
||||
{
|
||||
mediaType: 'video/mp4',
|
||||
uint8Array: new Uint8Array([50, 60, 70, 80]),
|
||||
base64: Buffer.from([50, 60, 70, 80]).toString('base64'),
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
} as unknown as Awaited<ReturnType<typeof aiModule.generateText>>)
|
||||
|
||||
const result = await mediaGenerationService.generateVideo({
|
||||
provider: 'google',
|
||||
modelId: 'gemini-omni-1.1-flash',
|
||||
apiKey: 'test-gemini-key',
|
||||
mode: 'image_to_video',
|
||||
prompt: 'Make the character wave',
|
||||
image: fakeImageBuffer,
|
||||
})
|
||||
|
||||
expect(mockGenerateText).toHaveBeenCalledTimes(1)
|
||||
const callArgs = mockGenerateText.mock.calls[0][0]
|
||||
const messages = callArgs.messages as Array<{
|
||||
role: string
|
||||
content: Array<{ type: string; mediaType?: string; data?: Buffer }>
|
||||
}>
|
||||
expect(messages).toBeDefined()
|
||||
expect(messages[0].content).toHaveLength(2)
|
||||
expect(messages[0].content[0].type).toBe('text')
|
||||
expect(messages[0].content[1].type).toBe('file')
|
||||
expect(messages[0].content[1].mediaType).toBe('image/png')
|
||||
expect(messages[0].content[1].data).toEqual(fakeImageBuffer)
|
||||
expect(result.buffer).toEqual(Buffer.from([50, 60, 70, 80]))
|
||||
})
|
||||
|
||||
it('should throw an error if interactions API returns no video file', async () => {
|
||||
vi.mocked(aiModule.generateText).mockResolvedValue({
|
||||
files: [],
|
||||
warnings: [],
|
||||
} as unknown as Awaited<ReturnType<typeof aiModule.generateText>>)
|
||||
|
||||
await expect(
|
||||
mediaGenerationService.generateVideo({
|
||||
provider: 'google',
|
||||
modelId: 'gemini-omni-1.1-flash',
|
||||
apiKey: 'test-gemini-key',
|
||||
mode: 'text_to_video',
|
||||
prompt: 'A video that fails to generate',
|
||||
}),
|
||||
).rejects.toThrow('No video binary data returned by Gemini Omni Flash')
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
import type { ThinkingLevel, ThinkingLevelMap } from '@earendil-works/pi-ai'
|
||||
|
||||
export type ModelsDevReasoningOption =
|
||||
| { type: 'toggle' }
|
||||
| {
|
||||
type: 'effort'
|
||||
values: Array<
|
||||
'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max' | 'default' | null
|
||||
>
|
||||
}
|
||||
| { type: 'budget_tokens'; min?: number; max?: number }
|
||||
|
||||
const THINKING_LEVELS: readonly ThinkingLevel[] = [
|
||||
'minimal',
|
||||
'low',
|
||||
'medium',
|
||||
'high',
|
||||
'xhigh',
|
||||
'max',
|
||||
]
|
||||
|
||||
/**
|
||||
* Converts models.dev verified effort values into Pi's selectable thinking levels.
|
||||
* Values without a Pi equivalent (`default` and JSON `null`) are intentionally
|
||||
* omitted.
|
||||
*/
|
||||
export function getEffortThinkingLevelMap(
|
||||
options: readonly ModelsDevReasoningOption[],
|
||||
): ThinkingLevelMap | undefined {
|
||||
const effortValues = options.flatMap((option) => (option.type === 'effort' ? option.values : []))
|
||||
if (effortValues.length === 0) return undefined
|
||||
|
||||
const supported = new Set(effortValues)
|
||||
if (!THINKING_LEVELS.some((level) => supported.has(level)) && !supported.has('none'))
|
||||
return undefined
|
||||
|
||||
const map: ThinkingLevelMap = { off: supported.has('none') ? 'none' : null }
|
||||
for (const level of THINKING_LEVELS) {
|
||||
map[level] = supported.has(level) ? level : null
|
||||
}
|
||||
return map
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { teamService } from '@shumai/core/src/team/team'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { providerService } from './provider'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import * as generateModelsModule from './generator/generate-models'
|
||||
import { ProviderConfigSerializable, providerModelSchema } from '@shumai/dtos'
|
||||
import { getBuiltinProvidersMap } from '@shumai/core/src/provider/builtin'
|
||||
import { z } from 'zod'
|
||||
@@ -191,4 +193,336 @@ describe('ProviderService', () => {
|
||||
const updatedAgent = updatedAgents.find((a) => a.id === agent!.id)
|
||||
expect(updatedAgent?.modelId).toBe(initialModelId)
|
||||
})
|
||||
|
||||
it('should support granular model CRUD operations and order by id desc', async () => {
|
||||
const team = await teamService.ensureDefaultTeam()
|
||||
const provider = await providerService.create(
|
||||
team.id,
|
||||
'test-granular-provider',
|
||||
{ api: 'openai-completions', apiKey: 'key1' },
|
||||
[],
|
||||
)
|
||||
|
||||
// 1. createModel
|
||||
const createdModel1 = await providerService.createModel(team.id, provider.id, {
|
||||
modelId: 'custom-model-1',
|
||||
name: 'Custom Model 1',
|
||||
config: {
|
||||
reasoning: true,
|
||||
input: ['text', 'image'],
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
cost: { input: 1.5, output: 5, cacheRead: 0.1, cacheWrite: 0.5 },
|
||||
},
|
||||
})
|
||||
expect(createdModel1.modelId).toBe('custom-model-1')
|
||||
expect(createdModel1.name).toBe('Custom Model 1')
|
||||
expect((createdModel1.config as { reasoning: boolean }).reasoning).toBe(true)
|
||||
|
||||
// Check duplicate modelId error
|
||||
await expect(
|
||||
providerService.createModel(team.id, provider.id, {
|
||||
modelId: 'custom-model-1',
|
||||
name: 'Duplicate',
|
||||
config: {
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 100000,
|
||||
maxTokens: 4000,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('already exists')
|
||||
|
||||
// 2. createModel 2
|
||||
const createdModel2 = await providerService.createModel(team.id, provider.id, {
|
||||
modelId: 'custom-model-2',
|
||||
name: 'Custom Model 2',
|
||||
config: {
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 100000,
|
||||
maxTokens: 4000,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
})
|
||||
|
||||
// 3. listModelsByProvider should return newest first (id desc)
|
||||
const models = await providerService.listModelsByProvider(team.id, provider.id)
|
||||
expect(models).toHaveLength(2)
|
||||
expect(models[0].id).toBe(createdModel2.id)
|
||||
expect(models[1].id).toBe(createdModel1.id)
|
||||
|
||||
// 4. updateModel
|
||||
const updatedModel = await providerService.updateModel(team.id, provider.id, createdModel1.id, {
|
||||
name: 'Updated Model 1',
|
||||
config: {
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 64000,
|
||||
maxTokens: 2048,
|
||||
cost: { input: 2, output: 8, cacheRead: 0.2, cacheWrite: 0.8 },
|
||||
},
|
||||
})
|
||||
expect(updatedModel.name).toBe('Updated Model 1')
|
||||
expect((updatedModel.config as { reasoning: boolean }).reasoning).toBe(false)
|
||||
expect((updatedModel.config as { cost: { input: number } }).cost.input).toBe(2)
|
||||
|
||||
// 5. update provider config and name without touching models
|
||||
const updatedProvider = await providerService.update(
|
||||
team.id,
|
||||
provider.id,
|
||||
{ api: 'openai-completions', apiKey: 'new-key' },
|
||||
undefined,
|
||||
'renamed-provider',
|
||||
)
|
||||
expect(updatedProvider.name).toBe('renamed-provider')
|
||||
expect(updatedProvider.config.apiKey).toBe('new-key')
|
||||
const modelsAfterProviderUpdate = await providerService.listModelsByProvider(
|
||||
team.id,
|
||||
provider.id,
|
||||
)
|
||||
expect(modelsAfterProviderUpdate).toHaveLength(2)
|
||||
|
||||
// 6. deleteModel
|
||||
await providerService.deleteModel(team.id, provider.id, createdModel2.id)
|
||||
const modelsAfterDelete = await providerService.listModelsByProvider(team.id, provider.id)
|
||||
expect(modelsAfterDelete).toHaveLength(1)
|
||||
expect(modelsAfterDelete[0].id).toBe(createdModel1.id)
|
||||
})
|
||||
|
||||
it('should propagate provider API protocol change to existing models', async () => {
|
||||
const team = await teamService.ensureDefaultTeam()
|
||||
|
||||
// 1. Create provider with openai-responses
|
||||
const provider = await providerService.create(
|
||||
team.id,
|
||||
'test-protocol-provider',
|
||||
{ api: 'openai-responses', apiKey: 'key-1' },
|
||||
[
|
||||
{
|
||||
modelId: 'm1',
|
||||
name: 'Model 1',
|
||||
config: {
|
||||
api: 'openai-responses',
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
},
|
||||
{
|
||||
modelId: 'm2',
|
||||
name: 'Model 2',
|
||||
config: {
|
||||
api: 'openai-responses',
|
||||
reasoning: true,
|
||||
input: ['text'],
|
||||
contextWindow: 64000,
|
||||
maxTokens: 2048,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
const initialModels = await providerService.listModelsByProvider(team.id, provider.id)
|
||||
expect(initialModels).toHaveLength(2)
|
||||
expect((initialModels[0].config as { api: string }).api).toBe('openai-responses')
|
||||
expect((initialModels[1].config as { api: string }).api).toBe('openai-responses')
|
||||
|
||||
// 2. Update provider global API protocol to anthropic-messages without passing models
|
||||
const updatedProvider = await providerService.update(
|
||||
team.id,
|
||||
provider.id,
|
||||
{ api: 'anthropic-messages', apiKey: 'key-2' },
|
||||
undefined,
|
||||
'test-protocol-provider',
|
||||
)
|
||||
expect(updatedProvider.config.api).toBe('anthropic-messages')
|
||||
|
||||
// 3. Verify that all existing models under this provider were updated
|
||||
const modelsAfterUpdate = await providerService.listModelsByProvider(team.id, provider.id)
|
||||
expect(modelsAfterUpdate).toHaveLength(2)
|
||||
expect((modelsAfterUpdate[0].config as { api: string }).api).toBe('anthropic-messages')
|
||||
expect((modelsAfterUpdate[1].config as { api: string }).api).toBe('anthropic-messages')
|
||||
})
|
||||
|
||||
it(
|
||||
'should check for updates and correctly segregate new vs existing providers',
|
||||
{ timeout: 30000 },
|
||||
async () => {
|
||||
// 1. Check updates with empty team: all discovered providers are new
|
||||
const emptyTeam = await prisma.team.create({
|
||||
data: { name: 'Empty Provider Team' },
|
||||
})
|
||||
const initialCheck = await providerService.checkUpdates(emptyTeam.id)
|
||||
expect(initialCheck.totalNewProviders).toBeGreaterThan(0)
|
||||
expect(initialCheck.totalNewModels).toBeGreaterThan(0)
|
||||
expect(initialCheck.providers.every((p) => p.isNewProvider)).toBe(true)
|
||||
|
||||
// 2. With default team (which has builtin providers initialized)
|
||||
const teamWithBuiltins = await teamService.ensureDefaultTeam()
|
||||
|
||||
// 3. Check updates after init: existing providers should have isNewProvider: false
|
||||
const afterInitCheck = await providerService.checkUpdates(teamWithBuiltins.id)
|
||||
const existing = afterInitCheck.providers.find((p) => !p.isNewProvider)
|
||||
if (existing) {
|
||||
expect(existing.isNewProvider).toBe(false)
|
||||
const dbModels = await providerService.listModelsByProvider(
|
||||
teamWithBuiltins.id,
|
||||
(await providerService.listByTeam(teamWithBuiltins.id)).find(
|
||||
(p) => p.name === existing.name,
|
||||
)!.id,
|
||||
)
|
||||
const dbModelIds = new Set(dbModels.map((m) => m.modelId))
|
||||
for (const m of existing.models) {
|
||||
expect(dbModelIds.has(m.modelId)).toBe(false)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
it('should throw an error when upstream catalog fetch fails in checkUpdates', async () => {
|
||||
const team = await teamService.ensureDefaultTeam()
|
||||
const spy = vi
|
||||
.spyOn(generateModelsModule, 'generateModels')
|
||||
.mockRejectedValueOnce(new Error('Failed to load models.dev catalog: 502 Bad Gateway'))
|
||||
|
||||
await expect(providerService.checkUpdates(team.id)).rejects.toThrow(
|
||||
'Failed to load models.dev catalog: 502 Bad Gateway',
|
||||
)
|
||||
spy.mockRestore()
|
||||
})
|
||||
|
||||
it('should throw an error when upstream catalog is empty in checkUpdates', async () => {
|
||||
const team = await teamService.ensureDefaultTeam()
|
||||
const spy = vi.spyOn(generateModelsModule, 'generateModels').mockResolvedValueOnce({
|
||||
providers: {},
|
||||
groupedProviders: {},
|
||||
sortedProviderIds: [],
|
||||
})
|
||||
|
||||
await expect(providerService.checkUpdates(team.id)).rejects.toThrow(
|
||||
'Failed to retrieve model catalog from upstream services',
|
||||
)
|
||||
spy.mockRestore()
|
||||
})
|
||||
|
||||
it('should apply sync by adding selected providers and models without modifying existing ones', async () => {
|
||||
const team = await teamService.ensureDefaultTeam()
|
||||
|
||||
// Create an existing provider with 1 model and a custom apiKey
|
||||
const existingProvider = await providerService.create(
|
||||
team.id,
|
||||
'test-existing-provider',
|
||||
{ api: 'openai-responses', apiKey: 'custom-api-key', baseUrl: 'https://example.com' },
|
||||
[
|
||||
{
|
||||
modelId: 'existing-m1',
|
||||
name: 'Existing Model 1',
|
||||
config: {
|
||||
api: 'openai-responses',
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
// Apply sync with 1 new provider and 1 new model on the existing provider
|
||||
const syncResult = await providerService.applySync(team.id, {
|
||||
providers: [
|
||||
{
|
||||
name: 'test-new-synced-provider',
|
||||
isNewProvider: true,
|
||||
config: {
|
||||
api: 'anthropic-messages',
|
||||
apiKey: 'ENV_SYNC_KEY',
|
||||
baseUrl: 'https://api.anthropic.com',
|
||||
},
|
||||
models: [
|
||||
{
|
||||
modelId: 'new-p-m1',
|
||||
name: 'New Provider Model 1',
|
||||
config: {
|
||||
api: 'anthropic-messages',
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
cost: { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'test-existing-provider',
|
||||
isNewProvider: false,
|
||||
config: { api: 'openai-responses', apiKey: 'should-not-override' },
|
||||
models: [
|
||||
{
|
||||
modelId: 'existing-m2',
|
||||
name: 'Newly Added Model 2',
|
||||
config: {
|
||||
api: 'openai-responses',
|
||||
reasoning: true,
|
||||
input: ['text'],
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
},
|
||||
// Duplicate model that already exists: should be ignored
|
||||
{
|
||||
modelId: 'existing-m1',
|
||||
name: 'Existing Model 1 Overwrite Attempt',
|
||||
config: {
|
||||
api: 'openai-responses',
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 999999,
|
||||
maxTokens: 999999,
|
||||
cost: { input: 9, output: 9, cacheRead: 9, cacheWrite: 9 },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
expect(syncResult.addedProviders).toBe(1)
|
||||
expect(syncResult.addedModels).toBe(2)
|
||||
|
||||
// Verify existing provider config was NOT modified
|
||||
const reloadedExisting = await providerService.getById(existingProvider.id)
|
||||
expect((reloadedExisting?.config as { apiKey: string }).apiKey).toBe('custom-api-key')
|
||||
|
||||
// Verify existing model was NOT modified
|
||||
const existingModels = await providerService.listModelsByProvider(team.id, existingProvider.id)
|
||||
expect(existingModels).toHaveLength(2)
|
||||
const m1 = existingModels.find((m) => m.modelId === 'existing-m1')
|
||||
expect(m1?.name).toBe('Existing Model 1')
|
||||
expect((m1?.config as { contextWindow: number }).contextWindow).toBe(128000)
|
||||
|
||||
// Verify newly added model on existing provider
|
||||
const m2 = existingModels.find((m) => m.modelId === 'existing-m2')
|
||||
expect(m2).toBeDefined()
|
||||
expect(m2?.name).toBe('Newly Added Model 2')
|
||||
|
||||
// Verify newly added provider
|
||||
const providers = await providerService.listByTeam(team.id)
|
||||
const newProvider = providers.find((p) => p.name === 'test-new-synced-provider')
|
||||
expect(newProvider).toBeDefined()
|
||||
expect(newProvider?.isBuiltin).toBe(true)
|
||||
expect((newProvider?.config as { apiKey: string }).apiKey).toBe('ENV_SYNC_KEY')
|
||||
|
||||
const newProviderModels = await providerService.listModelsByProvider(team.id, newProvider!.id)
|
||||
expect(newProviderModels).toHaveLength(1)
|
||||
expect(newProviderModels[0].modelId).toBe('new-p-m1')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,21 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { ProviderConfigSerializable, providerModelSchema } from '@shumai/dtos'
|
||||
import { getBuiltinProvidersMap } from '@shumai/core/src/provider/builtin'
|
||||
import {
|
||||
CreateModelRequest,
|
||||
ProviderConfigSerializable,
|
||||
providerModelSchema,
|
||||
SyncApplyRequest,
|
||||
SyncApplyResponse,
|
||||
SyncCheckResponse,
|
||||
SyncModelItem,
|
||||
SyncProviderItem,
|
||||
UpdateModelRequest,
|
||||
} from '@shumai/dtos'
|
||||
import {
|
||||
ENV_MAP,
|
||||
getBuiltinProvidersMap,
|
||||
PRIORITY_PROVIDERS,
|
||||
} from '@shumai/core/src/provider/builtin'
|
||||
import { generateModels } from '@shumai/core/src/provider/generator/generate-models'
|
||||
import { z } from 'zod'
|
||||
|
||||
type ProviderModel = z.infer<typeof providerModelSchema>
|
||||
@@ -74,7 +89,7 @@ export class ProviderService {
|
||||
providerId,
|
||||
provider: { teamId },
|
||||
},
|
||||
orderBy: { id: 'asc' },
|
||||
orderBy: { id: 'desc' },
|
||||
})
|
||||
}
|
||||
|
||||
@@ -82,7 +97,7 @@ export class ProviderService {
|
||||
teamId: string,
|
||||
name: string,
|
||||
config: ProviderConfigSerializable,
|
||||
models: ProviderModel[],
|
||||
models: ProviderModel[] = [],
|
||||
) {
|
||||
return this.prismaClient.provider.create({
|
||||
data: {
|
||||
@@ -105,60 +120,177 @@ export class ProviderService {
|
||||
teamId: string,
|
||||
id: string,
|
||||
config: ProviderConfigSerializable,
|
||||
models: ProviderModel[],
|
||||
models?: ProviderModel[],
|
||||
name?: string,
|
||||
) {
|
||||
return this.prismaClient.$transaction(async (tx) => {
|
||||
const existingModels = await tx.model.findMany({
|
||||
where: { providerId: id },
|
||||
})
|
||||
|
||||
const existingByModelId = new Map(existingModels.map((m) => [m.modelId, m]))
|
||||
const incomingModelIds = new Set(models.map((m) => m.modelId))
|
||||
|
||||
// Delete models that are no longer in the incoming list
|
||||
const modelsToDelete = existingModels.filter((m) => !incomingModelIds.has(m.modelId))
|
||||
if (modelsToDelete.length > 0) {
|
||||
await tx.model.deleteMany({
|
||||
where: {
|
||||
id: { in: modelsToDelete.map((m) => m.id) },
|
||||
},
|
||||
if (models !== undefined) {
|
||||
const existingModels = await tx.model.findMany({
|
||||
where: { providerId: id },
|
||||
})
|
||||
}
|
||||
|
||||
// Update existing models or create new ones
|
||||
for (const m of models) {
|
||||
const existing = existingByModelId.get(m.modelId)
|
||||
if (existing) {
|
||||
await tx.model.update({
|
||||
where: { id: existing.id },
|
||||
data: {
|
||||
name: m.name,
|
||||
config: m.config,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await tx.model.create({
|
||||
data: {
|
||||
providerId: id,
|
||||
modelId: m.modelId,
|
||||
name: m.name,
|
||||
config: m.config,
|
||||
const existingByModelId = new Map(existingModels.map((m) => [m.modelId, m]))
|
||||
const incomingModelIds = new Set(models.map((m) => m.modelId))
|
||||
|
||||
// Delete models that are no longer in the incoming list
|
||||
const modelsToDelete = existingModels.filter((m) => !incomingModelIds.has(m.modelId))
|
||||
if (modelsToDelete.length > 0) {
|
||||
await tx.model.deleteMany({
|
||||
where: {
|
||||
id: { in: modelsToDelete.map((m) => m.id) },
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Update existing models or create new ones
|
||||
for (const m of models) {
|
||||
const existing = existingByModelId.get(m.modelId)
|
||||
const modelConfig = {
|
||||
...m.config,
|
||||
api: m.config?.api || config.api,
|
||||
}
|
||||
if (existing) {
|
||||
await tx.model.update({
|
||||
where: { id: existing.id },
|
||||
data: {
|
||||
name: m.name,
|
||||
config: modelConfig,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await tx.model.create({
|
||||
data: {
|
||||
providerId: id,
|
||||
modelId: m.modelId,
|
||||
name: m.name,
|
||||
config: modelConfig,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
} else if (config.api) {
|
||||
// Propagate provider API protocol change to all existing models
|
||||
const existingModels = await tx.model.findMany({
|
||||
where: { providerId: id },
|
||||
})
|
||||
for (const m of existingModels) {
|
||||
const currentConfig = m.config
|
||||
if (currentConfig.api !== config.api) {
|
||||
await tx.model.update({
|
||||
where: { id: m.id },
|
||||
data: {
|
||||
config: {
|
||||
...currentConfig,
|
||||
api: config.api,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update provider config
|
||||
// Update provider config and optional name
|
||||
return tx.provider.update({
|
||||
where: { id, teamId },
|
||||
data: {
|
||||
config,
|
||||
...(name ? { name } : {}),
|
||||
},
|
||||
include: { models: true },
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async createModel(teamId: string, providerId: string, data: CreateModelRequest) {
|
||||
const provider = await this.prismaClient.provider.findUnique({
|
||||
where: { id: providerId, teamId },
|
||||
})
|
||||
if (!provider) {
|
||||
throw new Error('Provider not found')
|
||||
}
|
||||
|
||||
const existing = await this.prismaClient.model.findFirst({
|
||||
where: {
|
||||
providerId,
|
||||
modelId: data.modelId,
|
||||
},
|
||||
})
|
||||
if (existing) {
|
||||
throw new Error(`Model with id "${data.modelId}" already exists for this provider`)
|
||||
}
|
||||
|
||||
return this.prismaClient.model.create({
|
||||
data: {
|
||||
providerId,
|
||||
modelId: data.modelId,
|
||||
name: data.name ?? '',
|
||||
config: data.config,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async updateModel(
|
||||
teamId: string,
|
||||
providerId: string,
|
||||
modelDbId: string,
|
||||
data: UpdateModelRequest,
|
||||
) {
|
||||
const provider = await this.prismaClient.provider.findUnique({
|
||||
where: { id: providerId, teamId },
|
||||
})
|
||||
if (!provider) {
|
||||
throw new Error('Provider not found')
|
||||
}
|
||||
|
||||
const model = await this.prismaClient.model.findFirst({
|
||||
where: { id: modelDbId, providerId },
|
||||
})
|
||||
if (!model) {
|
||||
throw new Error('Model not found')
|
||||
}
|
||||
|
||||
if (data.modelId && data.modelId !== model.modelId) {
|
||||
const existingWithNewId = await this.prismaClient.model.findFirst({
|
||||
where: {
|
||||
providerId,
|
||||
modelId: data.modelId,
|
||||
},
|
||||
})
|
||||
if (existingWithNewId) {
|
||||
throw new Error(`Model with id "${data.modelId}" already exists for this provider`)
|
||||
}
|
||||
}
|
||||
|
||||
return this.prismaClient.model.update({
|
||||
where: { id: modelDbId },
|
||||
data: {
|
||||
...(data.modelId ? { modelId: data.modelId } : {}),
|
||||
...(data.name !== undefined ? { name: data.name } : {}),
|
||||
...(data.config ? { config: data.config } : {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async deleteModel(teamId: string, providerId: string, modelDbId: string) {
|
||||
const provider = await this.prismaClient.provider.findUnique({
|
||||
where: { id: providerId, teamId },
|
||||
})
|
||||
if (!provider) {
|
||||
throw new Error('Provider not found')
|
||||
}
|
||||
|
||||
const model = await this.prismaClient.model.findFirst({
|
||||
where: { id: modelDbId, providerId },
|
||||
})
|
||||
if (!model) {
|
||||
throw new Error('Model not found')
|
||||
}
|
||||
|
||||
return this.prismaClient.model.delete({
|
||||
where: { id: modelDbId },
|
||||
})
|
||||
}
|
||||
|
||||
async delete(teamId: string, id: string) {
|
||||
const provider = await this.prismaClient.provider.findUnique({
|
||||
where: { id, teamId },
|
||||
@@ -172,6 +304,181 @@ export class ProviderService {
|
||||
where: { id, teamId },
|
||||
})
|
||||
}
|
||||
|
||||
async checkUpdates(teamId: string): Promise<SyncCheckResponse> {
|
||||
const catalog = await generateModels({ strict: true, timeoutMs: 30000 })
|
||||
|
||||
if (!catalog || catalog.sortedProviderIds.length === 0) {
|
||||
throw new Error('Failed to retrieve model catalog from upstream services')
|
||||
}
|
||||
|
||||
const existingProviders = await this.prismaClient.provider.findMany({
|
||||
where: { teamId },
|
||||
include: { models: true },
|
||||
})
|
||||
|
||||
const existingProviderMap = new Map<string, (typeof existingProviders)[number]>()
|
||||
for (const p of existingProviders) {
|
||||
existingProviderMap.set(p.name, p)
|
||||
}
|
||||
|
||||
const items: SyncProviderItem[] = []
|
||||
let totalNewProviders = 0
|
||||
let totalNewModels = 0
|
||||
|
||||
for (const providerId of catalog.sortedProviderIds) {
|
||||
const modelsMap = catalog.providers[providerId]
|
||||
if (!modelsMap) continue
|
||||
const modelList = Object.values(modelsMap)
|
||||
if (modelList.length === 0) continue
|
||||
|
||||
const existingProvider = existingProviderMap.get(providerId)
|
||||
|
||||
if (!existingProvider) {
|
||||
const firstModel = modelList[0]
|
||||
const newModels: SyncModelItem[] = modelList.map((m) => ({
|
||||
modelId: m.id,
|
||||
name: m.name || m.id,
|
||||
config: {
|
||||
api: m.api as ProviderConfigSerializable['api'],
|
||||
reasoning: m.reasoning ?? false,
|
||||
input: (m.input ?? ['text']) as ('text' | 'image')[],
|
||||
contextWindow: m.contextWindow,
|
||||
maxTokens: m.maxTokens,
|
||||
cost: {
|
||||
input: m.cost?.input ?? 0,
|
||||
output: m.cost?.output ?? 0,
|
||||
cacheRead: m.cost?.cacheRead ?? 0,
|
||||
cacheWrite: m.cost?.cacheWrite ?? 0,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
items.push({
|
||||
name: providerId,
|
||||
isNewProvider: true,
|
||||
config: {
|
||||
api: firstModel.api as ProviderConfigSerializable['api'],
|
||||
baseUrl: firstModel.baseUrl,
|
||||
apiKey: ENV_MAP[providerId] || '',
|
||||
},
|
||||
models: newModels,
|
||||
})
|
||||
|
||||
totalNewProviders++
|
||||
totalNewModels += newModels.length
|
||||
} else {
|
||||
const existingModelIds = new Set(existingProvider.models.map((m) => m.modelId))
|
||||
const newModels: SyncModelItem[] = []
|
||||
|
||||
for (const m of modelList) {
|
||||
if (!existingModelIds.has(m.id)) {
|
||||
newModels.push({
|
||||
modelId: m.id,
|
||||
name: m.name || m.id,
|
||||
config: {
|
||||
api: m.api as ProviderConfigSerializable['api'],
|
||||
reasoning: m.reasoning ?? false,
|
||||
input: (m.input ?? ['text']) as ('text' | 'image')[],
|
||||
contextWindow: m.contextWindow,
|
||||
maxTokens: m.maxTokens,
|
||||
cost: {
|
||||
input: m.cost?.input ?? 0,
|
||||
output: m.cost?.output ?? 0,
|
||||
cacheRead: m.cost?.cacheRead ?? 0,
|
||||
cacheWrite: m.cost?.cacheWrite ?? 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (newModels.length > 0) {
|
||||
items.push({
|
||||
name: providerId,
|
||||
isNewProvider: false,
|
||||
config: existingProvider.config as ProviderConfigSerializable,
|
||||
models: newModels,
|
||||
})
|
||||
totalNewModels += newModels.length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items.sort((a, b) => {
|
||||
const indexA = PRIORITY_PROVIDERS.indexOf(a.name)
|
||||
const indexB = PRIORITY_PROVIDERS.indexOf(b.name)
|
||||
if (indexA !== -1 && indexB !== -1) return indexA - indexB
|
||||
if (indexA !== -1) return -1
|
||||
if (indexB !== -1) return 1
|
||||
return a.name.localeCompare(b.name)
|
||||
})
|
||||
|
||||
return {
|
||||
providers: items,
|
||||
totalNewProviders,
|
||||
totalNewModels,
|
||||
}
|
||||
}
|
||||
|
||||
async applySync(teamId: string, payload: SyncApplyRequest): Promise<SyncApplyResponse> {
|
||||
let addedProviders = 0
|
||||
let addedModels = 0
|
||||
|
||||
await this.prismaClient.$transaction(async (tx) => {
|
||||
for (const providerItem of payload.providers) {
|
||||
let providerId: string
|
||||
|
||||
if (providerItem.isNewProvider) {
|
||||
const existing = await tx.provider.findFirst({
|
||||
where: { teamId, name: providerItem.name },
|
||||
})
|
||||
if (!existing) {
|
||||
const created = await tx.provider.create({
|
||||
data: {
|
||||
teamId,
|
||||
name: providerItem.name,
|
||||
isBuiltin: true,
|
||||
config: providerItem.config,
|
||||
},
|
||||
})
|
||||
providerId = created.id
|
||||
addedProviders++
|
||||
} else {
|
||||
providerId = existing.id
|
||||
}
|
||||
} else {
|
||||
const existing = await tx.provider.findFirst({
|
||||
where: { teamId, name: providerItem.name },
|
||||
})
|
||||
if (!existing) continue
|
||||
providerId = existing.id
|
||||
}
|
||||
|
||||
for (const modelItem of providerItem.models) {
|
||||
const existingModel = await tx.model.findFirst({
|
||||
where: {
|
||||
providerId,
|
||||
modelId: modelItem.modelId,
|
||||
},
|
||||
})
|
||||
if (!existingModel) {
|
||||
await tx.model.create({
|
||||
data: {
|
||||
providerId,
|
||||
modelId: modelItem.modelId,
|
||||
name: modelItem.name || modelItem.modelId,
|
||||
config: modelItem.config,
|
||||
},
|
||||
})
|
||||
addedModels++
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return { addedProviders, addedModels }
|
||||
}
|
||||
}
|
||||
|
||||
export const providerService = new ProviderService()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user