mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-20 23:48:40 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7ed3bafdf | ||
|
|
9f76e61e23 | ||
|
|
7c297895e5 | ||
|
|
0f318b4050 | ||
|
|
1a46f63160 | ||
|
|
6cc90bf973 | ||
|
|
b7271777e0 | ||
|
|
c302c162a0 | ||
|
|
432be04a96 | ||
|
|
6c95d14fb3 | ||
|
|
3263580e89 | ||
|
|
f3da916905 | ||
|
|
fe8d404a44 | ||
|
|
0f9d213b1d | ||
|
|
f952318b34 | ||
|
|
43a2eced75 | ||
|
|
e894457275 | ||
|
|
4b1b64fe2b | ||
|
|
f8beb195dd | ||
|
|
a97996cb59 | ||
|
|
f870ca9b49 | ||
|
|
2e5ff281a8 | ||
|
|
de8decf113 | ||
|
|
dd57fb9ed2 | ||
|
|
9a640632aa | ||
|
|
e61641d7ec | ||
|
|
7ca8ba8b2a | ||
|
|
8805d5799a |
@@ -10,6 +10,10 @@ on:
|
||||
description: 'Docker image tag (e.g. feature-vaapi)'
|
||||
required: false
|
||||
type: string
|
||||
pr:
|
||||
description: 'Pull request number (e.g. 449)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
@@ -20,7 +24,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
id: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr && format('refs/pull/{0}/head', inputs.pr) || github.ref }}
|
||||
|
||||
# Set up Docker Buildx
|
||||
- name: Set up Docker Buildx
|
||||
@@ -51,8 +58,11 @@ jobs:
|
||||
crpi-000nkafzu6xvpwsc.cn-hongkong.personal.cr.aliyuncs.com/shumai-one/shumai
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=ref,event=branch
|
||||
type=ref,event=branch,enable=${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.pr && !inputs.tag) }}
|
||||
type=raw,value=pr-${{ inputs.pr }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.pr != '' && inputs.tag == '' }}
|
||||
type=raw,value=${{ inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.tag != '' }}
|
||||
labels: |
|
||||
org.opencontainers.image.revision=${{ steps.checkout.outputs.commit }}
|
||||
|
||||
# Build and push Docker image
|
||||
- name: Build and push Docker image
|
||||
|
||||
@@ -12,6 +12,49 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Changed
|
||||
|
||||
## [0.5.3] - 2026-10-16
|
||||
|
||||
### Added
|
||||
|
||||
* **cli**: Add administrative `shumai regenerate-embeddings` command to regenerate multimodal vector embeddings for all existing media files across active teams
|
||||
* **docker**: Provide three Docker Compose deployment tiers (Lite, with Embedding Server, and Full) with recommended memory specifications
|
||||
* **agent**: Save video chunk embeddings incrementally to the database as each chunk finishes, preserving completed progress on failure and resuming from where it left off on retry
|
||||
* **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing
|
||||
|
||||
### Fixed
|
||||
|
||||
### Changed
|
||||
|
||||
* **search**: Display search results in a responsive card grid with interactive hover video sprite scrubbing, semantic match chunk time range previews, and new-tab opening on click
|
||||
* **search**: Significantly improve semantic search response times by generating query text embeddings directly with in-memory caching instead of dispatching background tasks
|
||||
|
||||
## [0.5.2] - 2026-10-13
|
||||
|
||||
### Added
|
||||
|
||||
* **3d asset**: Add support for 3D model files (.glb, .gltf, .obj, .fbx, .stl, .dae, .usd, .usda, .usdc, .usdz) with automated turntable video rendering, hover sprite scrubbing, and an interactive 3D turntable viewer featuring 360° ruler scrubbing and degree-based markup comments
|
||||
* **kanban**: Related assets on a task are now clickable, taking you directly to that file or folder in its project
|
||||
* **settings**: Add an Integrations section to Team Settings with dedicated configuration for Gotenberg Document Converter, supporting connection testing and environment variable override protection
|
||||
|
||||
### Fixed
|
||||
|
||||
* **transcode**: Fix an issue where portrait photos from cameras and phones appeared sideways in thumbnails, previews, watermarked share links and annotated snapshots, and were listed with their width and height swapped
|
||||
|
||||
### Changed
|
||||
|
||||
## [0.5.1] - 2026-10-06
|
||||
|
||||
### Added
|
||||
|
||||
* **notification**: Notification cards are now clickable, taking you directly to the relevant file
|
||||
* **transcode**: Add hardware decoding and GPU scaling support for NVIDIA NVENC, Intel QuickSync Video (QSV), Rockchip (RKMPP), and VA-API (Intel and AMD GPUs), significantly speeding up video proxy and HLS rendition generation across supported platforms. Set `SHUMAI_HW_DECODE=false` to turn hardware decoding off
|
||||
* **transcode**: Add `dcraw_emu` fallback to generate WebP previews for camera RAW files without embedded JPEG previews
|
||||
|
||||
### Fixed
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
## [0.5.0] - 2026-10-02
|
||||
|
||||
### Added
|
||||
|
||||
@@ -52,6 +52,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
poppler-utils \
|
||||
imagemagick \
|
||||
fonts-noto-cjk \
|
||||
libraw-bin \
|
||||
libimage-exiftool-perl \
|
||||
&& . /etc/os-release \
|
||||
&& TARGETARCH=$(dpkg --print-architecture) \
|
||||
&& FFMPEG_VERSION="8.1.2-5" \
|
||||
|
||||
@@ -33,7 +33,9 @@ Shumai is an open-source Frame.io alternative built for creative teams to collab
|
||||
* **Custom Asset Metadata**: Define dynamic metadata fields tailored to your production pipeline.
|
||||
* **Share Link Watermarking**: Protect shared media with customizable text and image watermarks, and save watermark as reusable templates.
|
||||
* **Wide File Format Support**: Preview common creative file formats, including images (with PSD support), videos, audio, and documents, while allowing any file type to be uploaded and stored.
|
||||
* **3D Asset Turntable Previews**: Generate interactive 3D model turntable previews and videos for formats like GLB, FBX, OBJ, STL, USD, and DAE using [turntable-renderer](https://docs.shumai.one/using-shumai/3d-asset-support).
|
||||
* **Office & Document PDF Proxying via Gotenberg**: Convert Word, PowerPoint, Excel, Markdown, HTML, and CSV documents to high-quality PDF proxies for in-browser previewing using [Gotenberg](https://docs.shumai.one/using-shumai/gotenberg).
|
||||
* **Semantic Search**: Find assets by visual or conceptual meaning using vector embeddings. See [documentation](https://docs.shumai.one/using-shumai/semantic-search).
|
||||
* **NLE Export & Premiere Pro Plugin (Experimental)**: Export timestamped review comments to your favorite non-linear editor (Premiere Pro XML, DaVinci Resolve EDL, Final Cut Pro, Avid Media Composer) or use the experimental Premiere Pro UXP plugin to import media assets and auto-sync comments directly to sequence timeline markers. See [documentation](https://docs.shumai.one/using-shumai/nle-and-premiere).
|
||||
|
||||
But Shumai goes beyond being a Frame.io alternative. With modern models like GPT-Image-2 and Seedance 2.5, AI can already handle a surprising amount of creative work. That means you can create or update videos and images from a simple prompt without having to jump back into Photoshop, Premiere or Houdini. That is why Shumai includes a powerful agent system:
|
||||
@@ -46,7 +48,6 @@ But Shumai goes beyond being a Frame.io alternative. With modern models like GPT
|
||||
* **Model Context Protocol (MCP)**: Connect agents to external tools and APIs through remote MCP servers, with OAuth authentication and per-agent MCP access.
|
||||
* **Isolated Sandbox Execution**: Run agent-submitted scripts safely in a sandboxed environment.
|
||||
* **AI Metadata Autofill**: Automatically fill in custom metadata for new assets.
|
||||
* **Semantic Search**: Find assets by visual or conceptual meaning using vector embeddings (Gemini Embedding 2).
|
||||
* **Resource Quotas**: Set usage limits on AI tokens, costs, and tool execution per member, role, or team-wide, with real-time tracking.
|
||||
|
||||
Here is a short demo video showing what the Shumai agent can do:
|
||||
@@ -82,13 +83,25 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
2. Download the `docker-compose.yaml` file:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.yaml
|
||||
```
|
||||
2. Download your preferred `docker-compose.yaml` configuration based on your needs and available RAM:
|
||||
|
||||
* **Lite (Suggested RAM: 2 GB+)**: Minimal deployment containing Shumai and PostgreSQL database.
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.yaml
|
||||
```
|
||||
|
||||
* **With Embedding Server (Suggested RAM: 4 GB+)**: Adds local multimodal semantic search powered by `shumaiOne/embedding-server` (`EmbeddingGemma 2 740M`).
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.embedding.yaml
|
||||
```
|
||||
|
||||
* **Full (Suggested RAM: 8 GB+)**: All-in-one setup including `embedding-server`, `turntable-renderer` (3D previews), and `Gotenberg` (document PDF proxying).
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.full.yaml
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> By default, the standard `docker-compose.yaml` uses CPU software transcoding. If your server has a GPU (NVIDIA NVENC, Intel QuickSync, AMD/Intel VA-API, or Rockchip RKMPP) and you want to enable GPU hardware acceleration, check the [Hardware Transcoding Guide](https://docs.shumai.one/configuration/hardware-transcoding).
|
||||
> By default, the Docker Compose setups use CPU software transcoding. If your server has a GPU (NVIDIA NVENC, Intel QuickSync, AMD/Intel VA-API, or Rockchip RKMPP) and you want to enable GPU hardware acceleration, check the [Hardware Transcoding Guide](https://docs.shumai.one/configuration/hardware-transcoding).
|
||||
3. Configure environment variables (optional):
|
||||
|
||||
* `SHUMAI_SERVER_PORT` controls the port the Shumai server listens on. The default is `3000`.
|
||||
@@ -128,188 +141,7 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon
|
||||
```
|
||||
6. Open your browser and access Shumai at `http://localhost:3000` (or `http://<your-server-ip>:3000` for remote deployments).
|
||||
|
||||
### Option 2: Install via NPM / Package Manager
|
||||
|
||||
Shumai is published as `@shumai-one/shumai` on NPM. This option allows you to run Shumai globally or locally.
|
||||
|
||||
#### Step 1: Start PostgreSQL with pgvector
|
||||
|
||||
Shumai requires PostgreSQL with the `pgvector` extension. Start a pre-configured database container using Docker:
|
||||
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
-e POSTGRES_USER=shumai \
|
||||
-e POSTGRES_PASSWORD=shumai_password \
|
||||
-e POSTGRES_DB=shumai_db \
|
||||
-p 5432:5432 \
|
||||
-d pgvector/pgvector:pg18
|
||||
```
|
||||
|
||||
#### Step 2: Create a workspace folder
|
||||
|
||||
Create a dedicated directory to store your environment configuration and media files (which are saved in a `./data` directory by default):
|
||||
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
|
||||
#### Step 3: Install Platform-Specific Dependencies
|
||||
|
||||
#### Linux
|
||||
|
||||
Shumai requires the following system packages on Linux:
|
||||
|
||||
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
|
||||
- **`poppler`** (`poppler-utils`) – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
|
||||
- **`imagemagick`** – Used for PSD format transcoding and color profile conversion to sRGB.
|
||||
- **`bubblewrap`**, **`socat`**, and **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`) for process isolation, networking, and workspace search.
|
||||
|
||||
Install all required packages with one command:
|
||||
|
||||
> Ubuntu/Debian
|
||||
```bash
|
||||
sudo apt install -y ffmpeg poppler-utils imagemagick bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Fedora
|
||||
```bash
|
||||
sudo dnf install -y ffmpeg poppler-utils ImageMagick bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Arch Linux
|
||||
```bash
|
||||
sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> **Ubuntu 24.04+** restricts unprivileged user namespaces by default. This prevents `anthropic-experimental/sandbox-runtime` from using `bubblewrap` for sandbox isolation.
|
||||
>
|
||||
> To temporarily allow user namespaces:
|
||||
>
|
||||
> ```bash
|
||||
> sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
> ```
|
||||
>
|
||||
> Alternatively, configure an AppArmor profile that grants the required `userns` permission to the relevant binaries.
|
||||
|
||||
---
|
||||
|
||||
#### macOS
|
||||
|
||||
Shumai requires:
|
||||
|
||||
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
|
||||
- **`poppler`** – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
|
||||
- **`imagemagick`** – Used for PSD format transcoding and color profile conversion to sRGB.
|
||||
- **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`).
|
||||
|
||||
Install the required packages with Homebrew:
|
||||
|
||||
```bash
|
||||
brew install ffmpeg poppler imagemagick ripgrep
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### Windows (alpha)
|
||||
|
||||
> [!WARNING]
|
||||
> Windows support in anthropic-experimental/sandbox-runtime is **alpha and the design is in flux**.
|
||||
>
|
||||
> The current implementation provides:
|
||||
> - Network egress filtering via the **Windows Filtering Platform (WFP)**.
|
||||
> - File read/write denial via **ACL stamping**.
|
||||
>
|
||||
> However, it is **not a security boundary against a deliberately adversarial sandboxed process**.
|
||||
>
|
||||
> The sandbox implementation will be substantially revised in a future release, where sandboxed processes will run under a dedicated sandbox user account.
|
||||
|
||||
Shumai requires:
|
||||
|
||||
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
|
||||
- **`imagemagick`** – Used for PSD format transcoding and color profile conversion to sRGB.
|
||||
|
||||
Install required packages using your preferred package manager:
|
||||
|
||||
**winget**
|
||||
```powershell
|
||||
winget install Gyan.FFmpeg ImageMagick.ImageMagick
|
||||
```
|
||||
|
||||
**Chocolatey**
|
||||
```powershell
|
||||
choco install ffmpeg imagemagick
|
||||
```
|
||||
|
||||
#### Step 4: Install Shumai globally
|
||||
|
||||
Install Shumai globally using your preferred package manager:
|
||||
|
||||
```bash
|
||||
# NPM
|
||||
npm install -g @shumai-one/shumai
|
||||
|
||||
# PNPM
|
||||
pnpm add -g @shumai-one/shumai
|
||||
|
||||
# Bun
|
||||
bun add -g @shumai-one/shumai
|
||||
```
|
||||
|
||||
#### Step 5: Configure Environment Variables
|
||||
|
||||
Create a `.env` file in your workspace folder (`shumai/`) and add the following configuration:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
BETTER_AUTH_SECRET=ySxs7DxzHDZBbeeHNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
STORAGE_BACKEND=local
|
||||
SHUMAI_SERVER_PORT=3000
|
||||
AWS_ENDPOINT_URL_S3=http://localhost:3000
|
||||
# Optional: base URL for MCP OAuth redirects (defaults to BETTER_AUTH_URL)
|
||||
MCP_OAUTH_REDIRECT_BASE_URL=http://localhost:3000
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> `SHUMAI_SERVER_PORT` sets the port the server starts on, while `AWS_ENDPOINT_URL_S3` is used by the browser to build file upload URLs.
|
||||
> If deploying on a remote server (e.g. `http://123.456.7.8`) with a mapped port (e.g. `12345:3000` in docker-compose), set `AWS_ENDPOINT_URL_S3` to `http://123.456.7.8:12345`.
|
||||
> `MCP_OAUTH_REDIRECT_BASE_URL` is only needed when MCP servers use OAuth: the authorization redirect URI defaults to `{BETTER_AUTH_URL}/api/mcp/oauth/callback`; set it to an externally reachable URL if the app is served behind a proxy.
|
||||
|
||||
#### Step 6: Run Shumai
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **For Bun Users:** Since the published package binaries contain a `#!/usr/bin/env node` shebang, running the global command directly (e.g., `shumai`) will execute under Node.js. If you installed with Bun and want to run under the Bun runtime, you must prefix all commands with `bun run --bun` (e.g., `bun run --bun shumai`, `bun run --bun shumai -d`, `bun run --bun shumai stop`).
|
||||
|
||||
Start the application from your workspace folder:
|
||||
|
||||
```bash
|
||||
shumai
|
||||
```
|
||||
|
||||
Alternatively, you can run and manage Shumai in daemon mode:
|
||||
|
||||
- **Start in daemon mode**:
|
||||
```bash
|
||||
shumai -d
|
||||
```
|
||||
- **Stop Shumai**:
|
||||
```bash
|
||||
shumai stop
|
||||
```
|
||||
- **Restart Shumai**:
|
||||
```bash
|
||||
shumai restart
|
||||
```
|
||||
- **Show/tail logs**:
|
||||
```bash
|
||||
shumai logs
|
||||
```
|
||||
|
||||
On startup, Shumai will automatically run database migrations and start the web server at `http://localhost:3000`.
|
||||
|
||||
---
|
||||
|
||||
### Option 3: Run from Source (Development)
|
||||
### Option 2: Run from Source (Development)
|
||||
|
||||
To set up Shumai locally for development:
|
||||
|
||||
@@ -319,8 +151,23 @@ To set up Shumai locally for development:
|
||||
cd shumai
|
||||
bun install
|
||||
```
|
||||
2. Start the `pgvector` database container (as described in **Option 2, Step 1**).
|
||||
3. Create a `.env` file at the root of the workspace using the configuration from **Option 2, Step 5**.
|
||||
2. Start the `pgvector` database container:
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
-e POSTGRES_USER=shumai \
|
||||
-e POSTGRES_PASSWORD=shumai_password \
|
||||
-e POSTGRES_DB=shumai_db \
|
||||
-p 5432:5432 \
|
||||
-d pgvector/pgvector:pg18
|
||||
```
|
||||
3. Create a `.env` file at the root of the workspace:
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
BETTER_AUTH_SECRET=ySxs7DxzHDZBbeeHNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
STORAGE_BACKEND=local
|
||||
SHUMAI_SERVER_PORT=3000
|
||||
AWS_ENDPOINT_URL_S3=http://localhost:3000
|
||||
```
|
||||
4. Generate i18n translation files:
|
||||
```bash
|
||||
bun run i18n:compile
|
||||
@@ -336,6 +183,9 @@ To set up Shumai locally for development:
|
||||
bun run dev
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Looking to install Shumai globally via NPM or your package manager? See our [NPM Installation Guide](https://docs.shumai.one/getting-started/quickstart).
|
||||
|
||||
---
|
||||
|
||||
## Command Line Interface (CLI)
|
||||
|
||||
+56
-219
@@ -33,7 +33,9 @@ Shumai 是一个开源的 Frame.io 替代方案,专为创意团队打造,让
|
||||
* **自定义素材元数据**:可以按你的制作流程,自由定义动态元数据字段。
|
||||
* **分享链接水印**:通过可自定义的文字和图片水印保护共享媒体,并将水印配置保存为模板以便重复使用。
|
||||
* **丰富的文件格式支持**:支持预览常见创意文件格式,包括图片(支持 PSD)、视频、音频和文档,同时可上传和存储任意类型的文件。
|
||||
* **基于转盘渲染服务的 3D 资产预览**:通过集成 [turntable-renderer](https://docs.shumai.one/using-shumai/3d-asset-support) 支持为 GLB、FBX、OBJ、STL、USD 和 DAE 等 3D 模型生成高质量的交互式转盘预览与视频。
|
||||
* **基于 Gotenberg 的文档 PDF 转码代理**:通过集成 [Gotenberg](https://docs.shumai.one/using-shumai/gotenberg) 支持将 Word、PowerPoint、Excel、Markdown、HTML 和 CSV 文档高质量转换为 PDF 代理文件进行在线预览。
|
||||
* **语义搜索**:支持按画面内容或概念含义搜索素材,底层基于向量嵌入技术。详见[相关文档](https://docs.shumai.one/using-shumai/semantic-search)。
|
||||
* **非编剪辑导出与 Premiere 插件(实验性)**:支持将时间戳审阅评论导出至常用非编软件(Premiere Pro XML、DaVinci Resolve EDL、Final Cut Pro、Avid Media Composer),或使用实验性的 Premiere Pro 原生 UXP 插件导入媒体素材并将评论自动同步为时间轴标记。详见[相关文档](https://docs.shumai.one/using-shumai/nle-and-premiere)。
|
||||
|
||||
但 Shumai 不仅仅只满足于 Frame.io 的开源替代。随着 GPT-Image-2、Seedance 2.5 这类强大模型的问世,AI已经生成产品级的图片/视频。很多时候,你只要给一句提示词,就能直接生成或修改图片和视频,而不用再打开 Photoshop、Premiere 或者 Houdini。也正因为这样,Shumai 才把 AI Agent 做成了核心能力:
|
||||
@@ -44,7 +46,6 @@ Shumai 是一个开源的 Frame.io 替代方案,专为创意团队打造,让
|
||||
* **Model Context Protocol (MCP) 支持**:通过远程 MCP 服务端为 Agent 接入外部工具与 API,支持 OAuth 鉴权与按 Agent 独立分配访问权限。
|
||||
* **沙箱隔离执行**:Agent 生成的脚本会在隔离环境里运行,更安全。
|
||||
* **AI 自动补全元数据**:新素材上传后,系统可以自动帮你补一些元数据。
|
||||
* **语义搜索**:支持按画面内容或语义找素材,底层用的是向量 embedding(Gemini Embedding 2)。
|
||||
* **资源配额**:支持按成员、角色或全团队对 AI Token、成本和工具调用设置使用上限与实时用量监控。
|
||||
|
||||
下面是一个简短的 demo 视频,展示了 Shumai Agent 能做什么:
|
||||
@@ -98,10 +99,22 @@ https://gh-proxy.com/https://raw.githubusercontent.com/shumaiOne/shumai/main/REA
|
||||
|
||||
```
|
||||
|
||||
2. 下载 `docker-compose.yaml` 配置文件:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose-aliyun.yaml
|
||||
```
|
||||
2. 根据您的需求和机器可用内存,下载对应的 `docker-compose.yaml` 配置文件:
|
||||
|
||||
* **精简版(建议内存:2 GB+)**:包含 Shumai 核心应用与 PostgreSQL 数据库。
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose-aliyun.yaml
|
||||
```
|
||||
|
||||
* **含嵌入向量服务版(建议内存:4 GB+)**:新增基于 `shumaiOne/embedding-server` (`EmbeddingGemma 2 740M`) 的本地多模态语义搜索能力。
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose-aliyun.embedding.yaml
|
||||
```
|
||||
|
||||
* **完整版(建议内存:8 GB+)**:一站式全功能套件,包含 `embedding-server`、`turntable-renderer`(3D 模型交互转盘预览)与 `Gotenberg`(文档与表格 PDF 代理转码)。
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose-aliyun.full.yaml
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> 默认的 Docker Compose 配置使用基于 CPU 的软件转码。如果您的服务器配备了支持的显卡(NVIDIA NVENC、Intel QuickSync、AMD/Intel VA-API 或 Rockchip RKMPP)并需要开启 GPU 硬件加速转码,请参考 [硬件加速转码配置文档](https://docs.shumai.one/configuration/hardware-transcoding)。
|
||||
@@ -151,231 +164,55 @@ https://gh-proxy.com/https://raw.githubusercontent.com/shumaiOne/shumai/main/REA
|
||||
|
||||
5. 打开浏览器,访问 `http://localhost:3000` 即可开始使用(若是远程部署,请访问 `http://<您的服务器IP>:3000`)。
|
||||
|
||||
### 方案二:通过 NPM / 包管理器安装
|
||||
|
||||
通过 NPM 上的 `@shumai-one/shumai` 进行安装。
|
||||
|
||||
#### 第一步:启动带 pgvector 支持的 PostgreSQL
|
||||
|
||||
Shumai 依赖支持 `pgvector` 扩展的 PostgreSQL 数据库。您可以使用 Docker 快速启动一个预配置好的数据库容器:
|
||||
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
-e POSTGRES_USER=shumai \
|
||||
-e POSTGRES_PASSWORD=shumai_password \
|
||||
-e POSTGRES_DB=shumai_db \
|
||||
-p 5432:5432 \
|
||||
-d docker.m.daocloud.io/pgvector/pgvector:pg18
|
||||
|
||||
```
|
||||
|
||||
#### 第二步:创建工作区目录
|
||||
|
||||
创建一个专属目录用来存放环境配置和媒体文件(媒体文件默认会保存在 `./data` 目录下):
|
||||
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
|
||||
```
|
||||
|
||||
#### 步骤 3:安装平台依赖
|
||||
|
||||
#### Linux
|
||||
|
||||
在 Linux 上运行 Shumai 需要安装以下系统依赖:
|
||||
|
||||
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
|
||||
- **`poppler`** (`poppler-utils`) —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
|
||||
- **`imagemagick`** —— 用于 PSD 格式图像转码和 sRGB 色彩空间转换。
|
||||
- **`bubblewrap`**、**`socat`** 和 **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需,用于进程隔离、网络通信和工作区搜索。
|
||||
|
||||
可以使用以下命令一次性安装所有依赖:
|
||||
|
||||
> Ubuntu/Debian
|
||||
```bash
|
||||
sudo apt install -y ffmpeg poppler-utils imagemagick bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Fedora
|
||||
```bash
|
||||
sudo dnf install -y ffmpeg poppler-utils ImageMagick bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> Arch Linux
|
||||
```bash
|
||||
sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> **Ubuntu 24.04+** 默认限制非特权用户创建 User Namespace,这会导致 `anthropic-experimental/sandbox-runtime` 无法使用 `bubblewrap` 提供沙箱隔离。
|
||||
>
|
||||
> 如需临时启用 User Namespace,请执行:
|
||||
>
|
||||
> ```bash
|
||||
> sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
> ```
|
||||
>
|
||||
> 或者,配置 AppArmor Profile,为相关可执行文件授予所需的 `userns` 权限。
|
||||
|
||||
---
|
||||
|
||||
#### macOS
|
||||
|
||||
在 macOS 上运行 Shumai 需要:
|
||||
|
||||
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
|
||||
- **`poppler`** —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
|
||||
- **`imagemagick`** —— 用于 PSD 格式图像转码和 sRGB 色彩空间转换。
|
||||
- **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需。
|
||||
|
||||
使用 Homebrew 安装所需依赖:
|
||||
|
||||
```bash
|
||||
brew install ffmpeg poppler imagemagick ripgrep
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### Windows(Alpha)
|
||||
|
||||
> [!WARNING]
|
||||
> anthropic-experimental/sandbox-runtime 的 Windows 支持目前仍处于 **Alpha** 阶段,整体设计仍在持续调整中。
|
||||
>
|
||||
> 当前实现提供了以下能力:
|
||||
> - 通过 **Windows Filtering Platform (WFP)** 实现网络出口(egress)过滤。
|
||||
> - 通过 **ACL Stamping** 实现文件读写权限限制。
|
||||
>
|
||||
> **但这并不能作为针对恶意沙箱进程的安全边界(security boundary)**。
|
||||
>
|
||||
> 未来版本将对沙箱机制进行重大调整,届时沙箱进程将运行在独立的沙箱用户账户下。
|
||||
|
||||
在 Windows 上运行 Shumai 需要:
|
||||
|
||||
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
|
||||
- **`imagemagick`** —— 用于 PSD 格式图像转码和 sRGB 色彩空间转换。
|
||||
|
||||
使用你偏好的包管理器安装所需依赖:
|
||||
|
||||
**winget**
|
||||
```powershell
|
||||
winget install Gyan.FFmpeg ImageMagick.ImageMagick
|
||||
```
|
||||
|
||||
**Chocolatey**
|
||||
```powershell
|
||||
choco install ffmpeg imagemagick
|
||||
```
|
||||
|
||||
#### 第四步:全局安装 Shumai
|
||||
|
||||
使用您常用的包管理器进行全局安装:
|
||||
|
||||
```bash
|
||||
# 使用 NPM
|
||||
npm install -g @shumai-one/shumai
|
||||
|
||||
# 使用 PNPM
|
||||
pnpm add -g @shumai-one/shumai
|
||||
|
||||
# 使用 Bun
|
||||
bun add -g @shumai-one/shumai
|
||||
|
||||
```
|
||||
|
||||
#### 第五步:配置环境配置
|
||||
|
||||
在刚才创建的工作区目录(`shumai/`)下新建一个 `.env` 文件,并填入以下配置:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
BETTER_AUTH_SECRET=ySxs7DxzHDZBbeeHNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
STORAGE_BACKEND=local
|
||||
SHUMAI_SERVER_PORT=3000
|
||||
AWS_ENDPOINT_URL_S3=http://localhost:3000
|
||||
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> `SHUMAI_SERVER_PORT` 用于设置 Web 服务的启动端口,而 `AWS_ENDPOINT_URL_S3` 供浏览器端构建文件上传链接。
|
||||
> 如果您部署在远程服务器(如 `http://123.456.7.8`)并且做了端口映射(如 Docker 中的 `12345:3000`),请将 `AWS_ENDPOINT_URL_S3` 设置为 `http://123.456.7.8:12345`。
|
||||
|
||||
#### 第六步:启动 Shumai
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Bun 用户特别提示:** 由于发布的包二进制文件包含 `#!/usr/bin/env node` 头部声明(Shebang),直接在终端执行全局命令(例如 `shumai`)默认会通过 Node.js 运行。如果您是用 Bun 安装的并且希望调用 Bun 运行时,请在所有命令前加上 `bun run --bun` 前缀(例如:`bun run --bun shumai`、`bun run --bun shumai -d`、`bun run --bun shumai stop`)。
|
||||
|
||||
在您的工作区目录下执行以下命令启动应用:
|
||||
|
||||
```bash
|
||||
shumai
|
||||
|
||||
```
|
||||
|
||||
您也可以让 Shumai 在后台以守护进程(Daemon)模式运行和管理:
|
||||
|
||||
* **后台模式启动**:
|
||||
```bash
|
||||
shumai -d
|
||||
|
||||
```
|
||||
|
||||
|
||||
* **停止服务**:
|
||||
```bash
|
||||
shumai stop
|
||||
|
||||
```
|
||||
|
||||
|
||||
* **重启服务**:
|
||||
```bash
|
||||
shumai restart
|
||||
|
||||
```
|
||||
|
||||
|
||||
* **查看并跟踪实时日志**:
|
||||
```bash
|
||||
shumai logs
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
启动时,Shumai 会自动执行数据库迁移(Migration)并在 `http://localhost:3000` 拉起 Web 服务。
|
||||
|
||||
---
|
||||
|
||||
### 方案三:源码编译运行(开发环境)
|
||||
### 方案二:源码编译运行(开发环境)
|
||||
|
||||
如果您希望在本地参与 Shumai 的开发,请按以下步骤配置开发环境:
|
||||
|
||||
1. 克隆代码仓库并安装依赖项:
|
||||
```bash
|
||||
git clone [https://github.com/shumaiOne/shumai.git](https://github.com/shumaiOne/shumai.git)
|
||||
cd shumai
|
||||
bun install
|
||||
```bash
|
||||
git clone https://github.com/shumaiOne/shumai.git
|
||||
cd shumai
|
||||
bun install
|
||||
```
|
||||
|
||||
```
|
||||
2. 启动 `pgvector` 数据库容器:
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
-e POSTGRES_USER=shumai \
|
||||
-e POSTGRES_PASSWORD=shumai_password \
|
||||
-e POSTGRES_DB=shumai_db \
|
||||
-p 5432:5432 \
|
||||
-d docker.m.daocloud.io/pgvector/pgvector:pg18
|
||||
```
|
||||
|
||||
3. 在项目根目录下创建一个 `.env` 文件,填入以下基础配置:
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
BETTER_AUTH_SECRET=ySxs7DxzHDZBbeeHNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
STORAGE_BACKEND=local
|
||||
SHUMAI_SERVER_PORT=3000
|
||||
AWS_ENDPOINT_URL_S3=http://localhost:3000
|
||||
```
|
||||
|
||||
2. 启动 `pgvector` 数据库容器(参考**方案二中的第一步**)。
|
||||
3. 参照**方案二中的第五步**,在项目根目录下创建一个 `.env` 文件。
|
||||
4. 生成多语言国际化(i18n)代码:
|
||||
```bash
|
||||
bun run i18n:compile
|
||||
```
|
||||
> [!TIP]
|
||||
> 在前端开发过程中,可以在单独的终端窗口中运行 `bun run i18n:watch`,在修改 `packages/webui/messages/` 下的翻译文案时自动重新编译。
|
||||
```bash
|
||||
bun run i18n:compile
|
||||
```
|
||||
> [!TIP]
|
||||
> 在前端开发过程中,可以在单独的终端窗口中运行 `bun run i18n:watch`,在修改 `packages/webui/messages/` 下的翻译文案时自动重新编译。
|
||||
|
||||
5. 执行数据库结构迁移:
|
||||
```bash
|
||||
bun run db:migrate
|
||||
```
|
||||
```bash
|
||||
bun run db:migrate
|
||||
```
|
||||
|
||||
6. 启动本地开发服务器:
|
||||
```bash
|
||||
bun run dev
|
||||
```
|
||||
```bash
|
||||
bun run dev
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> 如需通过 NPM 或包管理器全局安装运行 Shumai,请参考[官方 NPM 安装指南](https://docs.shumai.one/getting-started/quickstart)。
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
export * from '@shumai/agent/src/workflows/agent-autofill'
|
||||
export * from '@shumai/agent/src/workflows/agent-chat'
|
||||
export * from '@shumai/agent/src/workflows/agent-embedding'
|
||||
export * from '@shumai/agent/src/workflows/query-embedding-for-search'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/premiere-uxp",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "commonjs",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -19,6 +19,7 @@ async function start() {
|
||||
process.env.BETTER_AUTH_URL = 'http://localhost:5200'
|
||||
process.env.NODE_ENV = 'development'
|
||||
process.env.SHUMAI_E2E = 'true'
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
|
||||
// Write variables to .env.e2e file so the teardown can load them
|
||||
const envContent = [`DATABASE_URL=${process.env.DATABASE_URL}`, `S3_BUCKET=${bucketName}`].join(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
@@ -12,9 +12,11 @@ import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { migrateLegacyAgentAvatars } from '@shumai/core/src/agent/migration'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
import { authService } from '@shumai/core/src/auth/auth'
|
||||
import { embeddingService } from '@shumai/core/src/search/embedding-service'
|
||||
import { sandboxService } from '@shumai/core'
|
||||
import { notificationJobService } from '@shumai/core/src/notification/notification-job'
|
||||
|
||||
@@ -47,6 +49,29 @@ if (resetCmdIndex !== -1) {
|
||||
}
|
||||
}
|
||||
|
||||
const regenCmdIndex = cliArgs.findIndex((arg) => arg === 'regenerate-embeddings')
|
||||
|
||||
if (regenCmdIndex !== -1) {
|
||||
try {
|
||||
console.log('🔄 Regenerating embeddings...')
|
||||
const result = await embeddingService.regenerateEmbeddings()
|
||||
console.log(
|
||||
`✅ Successfully queued ${result.tasksCreated} embedding tasks for ${result.totalAssets} assets.`,
|
||||
)
|
||||
if (result.skippedTeams.length > 0) {
|
||||
console.warn(
|
||||
`⚠️ Skipped ${result.skippedTeams.length} team(s) without an active embedding agent.`,
|
||||
)
|
||||
}
|
||||
console.log('The background workers will process the embedding tasks.')
|
||||
process.exit(0)
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Unknown error'
|
||||
console.error(`Error: ${message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
// Initialize workflows and activities for local executor mode
|
||||
initAgentWorkflows()
|
||||
@@ -58,6 +83,7 @@ async function run() {
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
await migrateLegacyAgentAvatars().catch(console.error)
|
||||
await agentService.ensureSystemEmbeddingAgents().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
notificationJobService.start()
|
||||
workflowService.start()
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
},
|
||||
"apps/agent": {
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"bin": {
|
||||
"shumai-agent": "./src/index.ts",
|
||||
},
|
||||
@@ -45,7 +45,7 @@
|
||||
},
|
||||
"apps/cli": {
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"bin": {
|
||||
"shumai-cli": "./src/index.ts",
|
||||
},
|
||||
@@ -60,7 +60,7 @@
|
||||
},
|
||||
"apps/premiere-uxp": {
|
||||
"name": "@shumai/premiere-uxp",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@shumai/api": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
@@ -114,7 +114,7 @@
|
||||
},
|
||||
"apps/transcode": {
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"bin": {
|
||||
"shumai-transcode": "./src/index.ts",
|
||||
},
|
||||
@@ -126,7 +126,7 @@
|
||||
},
|
||||
"apps/web": {
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/api": "workspace:*",
|
||||
@@ -146,12 +146,12 @@
|
||||
},
|
||||
"packages/agent": {
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.74",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@earendil-works/pi-ai": "0.82.1",
|
||||
"@google/genai": "2.19.0",
|
||||
"@google/genai": "2.28.0",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
@@ -161,6 +161,7 @@
|
||||
"@temporalio/workflow": "1.23.0",
|
||||
"adm-zip": "0.6.0",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"sharp": "0.35.4",
|
||||
"typebox": "1.3.22",
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.5.4",
|
||||
@@ -171,7 +172,7 @@
|
||||
},
|
||||
"packages/api": {
|
||||
"name": "@shumai/api",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@hono/zod-validator": "0.7.6",
|
||||
"@shumai/core": "workspace:*",
|
||||
@@ -185,7 +186,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@shumai/core",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@ai-sdk/alibaba": "2.0.38",
|
||||
"@ai-sdk/amazon-bedrock": "5.0.67",
|
||||
@@ -210,6 +211,7 @@
|
||||
"@aws-sdk/s3-request-presigner": "3.1121.0",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@earendil-works/pi-ai": "0.82.1",
|
||||
"@google/genai": "2.28.0",
|
||||
"@modelcontextprotocol/client": "2.0.0",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
@@ -247,7 +249,7 @@
|
||||
},
|
||||
"packages/db": {
|
||||
"name": "@shumai/db",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@prisma/adapter-pg": "7.10.0",
|
||||
@@ -263,7 +265,7 @@
|
||||
},
|
||||
"packages/dtos": {
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"zod": "4.5.4",
|
||||
@@ -271,10 +273,10 @@
|
||||
},
|
||||
"packages/e2e": {
|
||||
"name": "@shumai/e2e",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@google/genai": "2.19.0",
|
||||
"@google/genai": "2.28.0",
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -286,11 +288,11 @@
|
||||
},
|
||||
"packages/tableprinter": {
|
||||
"name": "@shumai/tableprinter",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
},
|
||||
"packages/transcode": {
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -304,7 +306,7 @@
|
||||
},
|
||||
"packages/webui": {
|
||||
"name": "@shumai/webui",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@base-ui/react": "1.7.0",
|
||||
"@dnd-kit/abstract": "0.5.0",
|
||||
@@ -406,7 +408,7 @@
|
||||
},
|
||||
"packages/workflow-core": {
|
||||
"name": "@shumai/workflow-core",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -652,7 +654,7 @@
|
||||
|
||||
"@floating-ui/utils": ["@floating-ui/utils@0.2.12", "", {}, "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww=="],
|
||||
|
||||
"@google/genai": ["@google/genai@2.19.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-IM3u1FjpOLyOPsPh0smGA6Ntj2olUWCPbTs/WQj7cA/2AKCGw/652W7IaySx0XtwuscFnMYBoraebQusSZ0zSQ=="],
|
||||
"@google/genai": ["@google/genai@2.28.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-hyNAJj+ywu3L46qlZod6KO/JvcT3QONz6VCiPevC/FWOjAD+LQb7DMLUoV8vzoLpt61Q0ruQX99iqbxEjPJWmw=="],
|
||||
|
||||
"@grpc/grpc-js": ["@grpc/grpc-js@1.14.4", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ=="],
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
services:
|
||||
postgres:
|
||||
image: docker.m.daocloud.io/pgvector/pgvector:pg18
|
||||
container_name: shumai_postgres
|
||||
environment:
|
||||
POSTGRES_USER: shumai
|
||||
POSTGRES_PASSWORD: shumai_password
|
||||
POSTGRES_DB: shumai_db
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/18/docker
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U shumai -d shumai_db']
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
embedding-server:
|
||||
image: shumaione/embedding-server:latest
|
||||
container_name: shumai_embedding
|
||||
restart: unless-stopped
|
||||
|
||||
shumai:
|
||||
image: crpi-000nkafzu6xvpwsc.cn-hongkong.personal.cr.aliyuncs.com/shumai-one/shumai:latest
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
- seccomp=unconfined
|
||||
container_name: shumai_app
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
# Hardware Acceleration (NVENC / QuickSync / VA-API / RKMPP)
|
||||
# Uncomment the extends section below for hardware acceleration:
|
||||
# extends:
|
||||
# file: hwaccel.transcoding.yaml
|
||||
# service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_URL: postgres://shumai:shumai_password@shumai_postgres:5432/shumai_db?search_path=public&sslmode=disable
|
||||
|
||||
# Server Configuration
|
||||
SHUMAI_SERVER_PORT: 3000
|
||||
|
||||
# S3 Configuration
|
||||
STORAGE_BACKEND: local
|
||||
AWS_ENDPOINT_URL_S3: http://localhost:3000
|
||||
|
||||
# Better Auth
|
||||
BETTER_AUTH_SECRET: qSxs9DxzHDZBbeeTNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
|
||||
# Sandbox Configuration
|
||||
ENABLE_WEAKER_NESTED_SANDBOX: 'true'
|
||||
|
||||
# Embedding Configuration
|
||||
EMBEDDING_PROVIDER: gemma-embedding-2-openai
|
||||
GEMMA_EMBEDDING_OPENAI_BASE_URL: http://embedding-server:9379/v1
|
||||
ENABLE_EMBEDDING_AGENT: 'true'
|
||||
CONCURRENCY_AGENT: 1
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
embedding-server:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,89 @@
|
||||
services:
|
||||
postgres:
|
||||
image: docker.m.daocloud.io/pgvector/pgvector:pg18
|
||||
container_name: shumai_postgres
|
||||
environment:
|
||||
POSTGRES_USER: shumai
|
||||
POSTGRES_PASSWORD: shumai_password
|
||||
POSTGRES_DB: shumai_db
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/18/docker
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U shumai -d shumai_db']
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
embedding-server:
|
||||
image: shumaione/embedding-server:latest
|
||||
container_name: shumai_embedding
|
||||
restart: unless-stopped
|
||||
|
||||
turntable-renderer:
|
||||
image: shumaione/turntable-renderer:latest
|
||||
container_name: shumai_turntable
|
||||
restart: unless-stopped
|
||||
|
||||
gotenberg:
|
||||
image: gotenberg/gotenberg:8
|
||||
container_name: shumai_gotenberg
|
||||
restart: unless-stopped
|
||||
|
||||
shumai:
|
||||
image: crpi-000nkafzu6xvpwsc.cn-hongkong.personal.cr.aliyuncs.com/shumai-one/shumai:latest
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
- seccomp=unconfined
|
||||
container_name: shumai_app
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
# Hardware Acceleration (NVENC / QuickSync / VA-API / RKMPP)
|
||||
# Uncomment the extends section below for hardware acceleration:
|
||||
# extends:
|
||||
# file: hwaccel.transcoding.yaml
|
||||
# service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_URL: postgres://shumai:shumai_password@shumai_postgres:5432/shumai_db?search_path=public&sslmode=disable
|
||||
|
||||
# Server Configuration
|
||||
SHUMAI_SERVER_PORT: 3000
|
||||
|
||||
# S3 Configuration
|
||||
STORAGE_BACKEND: local
|
||||
AWS_ENDPOINT_URL_S3: http://localhost:3000
|
||||
|
||||
# Better Auth
|
||||
BETTER_AUTH_SECRET: qSxs9DxzHDZBbeeTNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
|
||||
# Sandbox Configuration
|
||||
ENABLE_WEAKER_NESTED_SANDBOX: 'true'
|
||||
|
||||
# Embedding Configuration
|
||||
EMBEDDING_PROVIDER: gemma-embedding-2-openai
|
||||
GEMMA_EMBEDDING_OPENAI_BASE_URL: http://embedding-server:9379/v1
|
||||
ENABLE_EMBEDDING_AGENT: 'true'
|
||||
CONCURRENCY_AGENT: 1
|
||||
|
||||
# 3D Turntable Renderer Configuration
|
||||
TURNTABLE_RENDERER_URL: http://turntable-renderer:3000
|
||||
|
||||
# Gotenberg Document Converter Configuration
|
||||
GOTENBERG_URL: http://gotenberg:3000
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
embedding-server:
|
||||
condition: service_started
|
||||
turntable-renderer:
|
||||
condition: service_started
|
||||
gotenberg:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,69 @@
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
container_name: shumai_postgres
|
||||
environment:
|
||||
POSTGRES_USER: shumai
|
||||
POSTGRES_PASSWORD: shumai_password
|
||||
POSTGRES_DB: shumai_db
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/18/docker
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U shumai -d shumai_db']
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
embedding-server:
|
||||
image: shumaione/embedding-server:latest
|
||||
container_name: shumai_embedding
|
||||
restart: unless-stopped
|
||||
|
||||
shumai:
|
||||
image: shumaione/shumai:latest
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
- seccomp=unconfined
|
||||
container_name: shumai_app
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
# Hardware Acceleration (NVENC / QuickSync / VA-API / RKMPP)
|
||||
# Uncomment the extends section below for hardware acceleration:
|
||||
# extends:
|
||||
# file: hwaccel.transcoding.yaml
|
||||
# service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_URL: postgres://shumai:shumai_password@shumai_postgres:5432/shumai_db?search_path=public&sslmode=disable
|
||||
|
||||
# Server Configuration
|
||||
SHUMAI_SERVER_PORT: 3000
|
||||
|
||||
# S3 Configuration
|
||||
STORAGE_BACKEND: local
|
||||
AWS_ENDPOINT_URL_S3: http://localhost:3000
|
||||
|
||||
# Better Auth
|
||||
BETTER_AUTH_SECRET: qSxs9DxzHDZBbeeTNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
|
||||
# Sandbox Configuration
|
||||
ENABLE_WEAKER_NESTED_SANDBOX: 'true'
|
||||
|
||||
# Embedding Configuration
|
||||
EMBEDDING_PROVIDER: gemma-embedding-2-openai
|
||||
GEMMA_EMBEDDING_OPENAI_BASE_URL: http://embedding-server:9379/v1
|
||||
ENABLE_EMBEDDING_AGENT: 'true'
|
||||
CONCURRENCY_AGENT: 1
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
embedding-server:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,89 @@
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
container_name: shumai_postgres
|
||||
environment:
|
||||
POSTGRES_USER: shumai
|
||||
POSTGRES_PASSWORD: shumai_password
|
||||
POSTGRES_DB: shumai_db
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/18/docker
|
||||
healthcheck:
|
||||
test: ['CMD-SHELL', 'pg_isready -U shumai -d shumai_db']
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
embedding-server:
|
||||
image: shumaione/embedding-server:latest
|
||||
container_name: shumai_embedding
|
||||
restart: unless-stopped
|
||||
|
||||
turntable-renderer:
|
||||
image: shumaione/turntable-renderer:latest
|
||||
container_name: shumai_turntable
|
||||
restart: unless-stopped
|
||||
|
||||
gotenberg:
|
||||
image: gotenberg/gotenberg:8
|
||||
container_name: shumai_gotenberg
|
||||
restart: unless-stopped
|
||||
|
||||
shumai:
|
||||
image: shumaione/shumai:latest
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
security_opt:
|
||||
- apparmor=unconfined
|
||||
- seccomp=unconfined
|
||||
container_name: shumai_app
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
# Hardware Acceleration (NVENC / QuickSync / VA-API / RKMPP)
|
||||
# Uncomment the extends section below for hardware acceleration:
|
||||
# extends:
|
||||
# file: hwaccel.transcoding.yaml
|
||||
# service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_URL: postgres://shumai:shumai_password@shumai_postgres:5432/shumai_db?search_path=public&sslmode=disable
|
||||
|
||||
# Server Configuration
|
||||
SHUMAI_SERVER_PORT: 3000
|
||||
|
||||
# S3 Configuration
|
||||
STORAGE_BACKEND: local
|
||||
AWS_ENDPOINT_URL_S3: http://localhost:3000
|
||||
|
||||
# Better Auth
|
||||
BETTER_AUTH_SECRET: qSxs9DxzHDZBbeeTNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
|
||||
# Sandbox Configuration
|
||||
ENABLE_WEAKER_NESTED_SANDBOX: 'true'
|
||||
|
||||
# Embedding Configuration
|
||||
EMBEDDING_PROVIDER: gemma-embedding-2-openai
|
||||
GEMMA_EMBEDDING_OPENAI_BASE_URL: http://embedding-server:9379/v1
|
||||
ENABLE_EMBEDDING_AGENT: 'true'
|
||||
CONCURRENCY_AGENT: 1
|
||||
|
||||
# 3D Turntable Renderer Configuration
|
||||
TURNTABLE_RENDERER_URL: http://turntable-renderer:3000
|
||||
|
||||
# Gotenberg Document Converter Configuration
|
||||
GOTENBERG_URL: http://gotenberg:3000
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
embedding-server:
|
||||
condition: service_started
|
||||
turntable-renderer:
|
||||
condition: service_started
|
||||
gotenberg:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -3,6 +3,54 @@ title: "Changelog"
|
||||
description: "Recent updates, new features, and improvements in Shumai"
|
||||
---
|
||||
|
||||
<Update label="v0.5.3" description="October 16, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
* **cli**: Add administrative `shumai regenerate-embeddings` command to regenerate multimodal vector embeddings for all existing media files across active teams
|
||||
* **docker**: Provide three Docker Compose deployment tiers (Lite, with Embedding Server, and Full) with recommended memory specifications
|
||||
* **agent**: Save video chunk embeddings incrementally to the database as each chunk finishes, preserving completed progress on failure and resuming from where it left off on retry
|
||||
* **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing
|
||||
|
||||
### Fixed
|
||||
|
||||
### Changed
|
||||
|
||||
* **search**: Display search results in a responsive card grid with interactive hover video sprite scrubbing, semantic match chunk time range previews, and new-tab opening on click
|
||||
* **search**: Significantly improve semantic search response times by generating query text embeddings directly with in-memory caching instead of dispatching background tasks
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="v0.5.2" description="October 13, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
* **3d asset**: Add support for 3D model files (.glb, .gltf, .obj, .fbx, .stl, .dae, .usd, .usda, .usdc, .usdz) with automated turntable video rendering, hover sprite scrubbing, and an interactive 3D turntable viewer featuring 360° ruler scrubbing and degree-based markup comments
|
||||
* **kanban**: Related assets on a task are now clickable, taking you directly to that file or folder in its project
|
||||
* **settings**: Add an Integrations section to Team Settings with dedicated configuration for Gotenberg Document Converter, supporting connection testing and environment variable override protection
|
||||
|
||||
### Fixed
|
||||
|
||||
* **transcode**: Fix an issue where portrait photos from cameras and phones appeared sideways in thumbnails, previews, watermarked share links and annotated snapshots, and were listed with their width and height swapped
|
||||
|
||||
### Changed
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="v0.5.1" description="October 6, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
* **notification**: Notification cards are now clickable, taking you directly to the relevant file
|
||||
* **transcode**: Add hardware decoding and GPU scaling support for NVIDIA NVENC, Intel QuickSync Video (QSV), Rockchip (RKMPP), and VA-API (Intel and AMD GPUs), significantly speeding up video proxy and HLS rendition generation across supported platforms. Set `SHUMAI_HW_DECODE=false` to turn hardware decoding off
|
||||
* **transcode**: Add `dcraw_emu` fallback to generate WebP previews for camera RAW files without embedded JPEG previews
|
||||
|
||||
### Fixed
|
||||
|
||||
### Changed
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="v0.5.0" description="October 2, 2026">
|
||||
|
||||
### Added
|
||||
|
||||
@@ -66,12 +66,24 @@ For example, if you deploy Shumai on a remote server at `http://123.456.7.8` and
|
||||
|
||||
Configures whether background transcoding and AI agents are executed locally or offloaded to a Temporal cluster.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------------- | :------------------------------------------------------------------------------------------------- | :------------------------------ |
|
||||
| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` |
|
||||
| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) |
|
||||
| `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) |
|
||||
| `EMBEDDING_CHUNK_OVERLAP` | The duration in seconds to overlap video chunks when generating video embeddings. | `5.0` (default) |
|
||||
| Variable | Description | Example / Default |
|
||||
| :---------------------- | :------------------------------------------------------------------------------------------------- | :------------------------------ |
|
||||
| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` |
|
||||
| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) |
|
||||
| `CONCURRENCY_AGENT` | Maximum concurrent AI tasks executed by workers. Set to `1` when running local `embedding-server`. | `5` (default) |
|
||||
| `CONCURRENCY_TRANSCODE` | Maximum concurrent media transcoding tasks executed by workers. | `1` (default) |
|
||||
|
||||
---
|
||||
|
||||
## Embedding Config
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :-------------------------------- | :------------------------------------------------------------------------------------------------------------ | :-------------------------------- |
|
||||
| `EMBEDDING_PROVIDER` | Multimodal embedding provider. Options: `gemini-embedding-2`, `gemma-embedding-2-openai`. | `gemini-embedding-2` (default) |
|
||||
| `GEMMA_EMBEDDING_OPENAI_BASE_URL` | Base URL of the OpenAI-compatible LiteRT-LM embedding server. | `http://embedding-server:9379/v1` |
|
||||
| `ENABLE_EMBEDDING_AGENT` | When set to `true`, automatically provisions and enables the system embedding agent for all teams on startup. | `false` (default) |
|
||||
| `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) |
|
||||
| `EMBEDDING_CHUNK_OVERLAP` | The duration in seconds to overlap video chunks when generating video embeddings. | `5.0` (default) |
|
||||
|
||||
## Sandbox Config
|
||||
|
||||
@@ -99,3 +111,29 @@ API keys to AI providers on startup.
|
||||
| `GEMINI_API_KEY` | API key for Google Gemini provider. | `AIzaSy...` |
|
||||
| `OPENAI_API_KEY` | API key for OpenAI provider. | `sk-proj-...` |
|
||||
| `ANTHROPIC_API_KEY` | API key for Anthropic provider. | `sk-ant-...` |
|
||||
|
||||
---
|
||||
|
||||
## External Integrations Config
|
||||
|
||||
Environment variables can configure external microservices globally. When set, these environment variables take precedence over any team-level settings configured in **Team Settings → Integrations**, and lock the corresponding settings fields in the WebUI to read-only.
|
||||
|
||||
### Gotenberg Document Converter
|
||||
|
||||
Used to convert Office documents, HTML, Markdown, and CSV files into high-fidelity PDF proxies for in-browser previewing. See the [Gotenberg Guide](/using-shumai/gotenberg) for setup instructions.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------------------ | :----------------------------------------------------------------------- | :---------------------- |
|
||||
| `GOTENBERG_URL` | Base URL of the external Gotenberg conversion API. | `http://gotenberg:3000` |
|
||||
| `GOTENBERG_BASIC_AUTH_USERNAME` | Username when Basic Authentication is enabled on the Gotenberg instance. | `admin` |
|
||||
| `GOTENBERG_BASIC_AUTH_PASSWORD` | Password when Basic Authentication is enabled on the Gotenberg instance. | `secret` |
|
||||
|
||||
### 3D Turntable Renderer
|
||||
|
||||
Used to convert 3D model files (GLB, OBJ, FBX, USD, STL, DAE) into interactive 360° turntable video proxies, posters, and hover sprites. See the [3D Asset Support Guide](/using-shumai/3d-asset-support) for setup instructions.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :--------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | :------------------------------- |
|
||||
| `TURNTABLE_RENDERER_URL` | Base URL of the external turntable-renderer service. | `http://turntable-renderer:3000` |
|
||||
| `TURNTABLE_RENDERER_BASIC_AUTH_USERNAME` | Username when Basic Authentication is enabled on the turntable renderer (alias: `TURNTABLE_RENDERER_USERNAME`). | `admin` |
|
||||
| `TURNTABLE_RENDERER_BASIC_AUTH_PASSWORD` | Password when Basic Authentication is enabled on the turntable renderer (alias: `TURNTABLE_RENDERER_PASSWORD`). | `secret` |
|
||||
|
||||
@@ -55,6 +55,20 @@ Shumai includes a built-in usability probe that tests hardware encoders at start
|
||||
You should see `card0` and `renderD128` (or similar).
|
||||
2. Shumai's Docker image automatically adjusts group permissions on container startup so the non-root application user can access host `/dev/dri` devices without permission errors.
|
||||
|
||||
#### GPU decoding
|
||||
|
||||
With VA-API, Shumai decodes and resizes video on the GPU as well as encoding it, so frames never leave GPU memory. This is much faster than decoding on the CPU, especially for 4K sources on integrated graphics. (Other backends currently decode on the CPU and encode on the GPU.)
|
||||
|
||||
Some videos automatically use CPU decoding with GPU encoding instead:
|
||||
|
||||
- Codecs your GPU cannot decode, such as HEVC or 10-bit video on older Intel chips. Shumai tries the GPU first and retries on the CPU within a second.
|
||||
- Rotated videos, such as portrait phone recordings, so they keep the correct orientation.
|
||||
- Watermarked share links and HDR sources.
|
||||
|
||||
If a GPU-decode attempt stops making progress for 2 minutes (some driver bugs make ffmpeg hang instead of failing), Shumai stops it and retries on the CPU. Change the limit with `SHUMAI_HW_DECODE_STALL_TIMEOUT` (in seconds).
|
||||
|
||||
To always decode on the CPU, set `SHUMAI_HW_DECODE=false` on the `shumai` service.
|
||||
|
||||
### WSL2 (Windows)
|
||||
|
||||
- Ensure you are running Windows 11 with WSL2 and have installed the appropriate GPU drivers on the Windows host.
|
||||
|
||||
+3
-1
@@ -44,7 +44,9 @@
|
||||
"using-shumai/collections",
|
||||
"using-shumai/share-links",
|
||||
"using-shumai/nle-and-premiere",
|
||||
"using-shumai/gotenberg"
|
||||
"using-shumai/3d-asset-support",
|
||||
"using-shumai/gotenberg",
|
||||
"using-shumai/embedding-server"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -44,6 +44,9 @@ To handle media processing and security isolation, the host machine must have th
|
||||
- **FFmpeg & FFprobe**: Required for video transcoding and metadata extraction.
|
||||
- _macOS_: `brew install ffmpeg`
|
||||
- _Linux (Ubuntu/Debian)_: `sudo apt-get install ffmpeg`
|
||||
- **dcraw_emu (LibRaw)**: Required for camera RAW image decoding and preview generation.
|
||||
- _macOS_: `brew install libraw`
|
||||
- _Linux (Ubuntu/Debian)_: `sudo apt-get install libraw-bin`
|
||||
- **Agent Sandbox Utilities**: Used to run custom automation scripts securely.
|
||||
- _macOS_: `brew install ripgrep`
|
||||
- _Linux (Ubuntu/Debian)_: `sudo apt-get install bubblewrap socat ripgrep`
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
---
|
||||
title: 3D Asset Support with Turntable Renderer
|
||||
description: Render interactive 3D model turntable previews, sprite scrubbing, and video proxies using turntable-renderer.
|
||||
---
|
||||
|
||||
3D models and mesh assets cannot always be rendered consistently or inspected reliably across diverse client web browsers and mobile devices. Shumai integrates with [turntable-renderer](https://github.com/shumaiOne/turntable-renderer) — a standalone service that executes isolated, headless Blender 5.2 LTS subprocesses — to convert 3D model uploads into standardized, high-fidelity turntable previews.
|
||||
|
||||
Turntable Renderer automatically produces:
|
||||
|
||||
- **1080×1080 Turntable Video (`turntable.mp4`)**: A 4-second, 24-frame, 6 fps video rotating the model 360° (15° per frame) clockwise under dark studio lighting with a pure black background.
|
||||
- **Poster Image (`poster.webp`)**: A crisp still poster for grid cards and immediate file display.
|
||||
- **Hover Sprite Sheet (`sprite.webp`)**: A fast scrubbing sheet allowing users to rotate and inspect the model directly from the file browser on hover.
|
||||
- **Interactive 3D Viewer**: A specialized web viewer featuring 360° ruler scrubbing, rotation angle control (0° to 345°), playback controls, and angle-specific review comments and visual markups.
|
||||
|
||||
---
|
||||
|
||||
## Supported File Types
|
||||
|
||||
When turntable-renderer is configured, Shumai automatically generates 3D turntable proxies for the following formats:
|
||||
|
||||
| Format / Standard | Extensions |
|
||||
| --------------------------- | --------------------------------- |
|
||||
| Binary glTF | `.glb` |
|
||||
| Universal Scene Description | `.usd`, `.usda`, `.usdc`, `.usdz` |
|
||||
| Autodesk FBX | `.fbx` |
|
||||
| Wavefront OBJ | `.obj` |
|
||||
| Stereolithography (STL) | `.stl` |
|
||||
| COLLADA | `.dae` |
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
Shumai supports configuring the turntable renderer either through the **WebUI** or via **Environment Variables**.
|
||||
|
||||
### 1. WebUI (Team Settings → Integrations)
|
||||
|
||||
Team owners can configure the turntable renderer directly from the web interface:
|
||||
|
||||
1. Navigate to **Team Settings** from the sidebar.
|
||||
2. Select the **Integrations** tab.
|
||||
3. Under **3D Turntable Renderer**, enter the **Server URL** (e.g. `http://turntable-renderer:3000` or `http://localhost:3003`).
|
||||
4. Enter the **Username** and **Password** if Basic Authentication is enabled on the service.
|
||||
5. Click **Test Connection** to verify that Shumai can communicate with the renderer and check the detected Blender version.
|
||||
6. Click **Save Changes**.
|
||||
|
||||
### 2. Environment Variables
|
||||
|
||||
You can also configure turntable-renderer globally across the system using environment variables on the machine running Shumai or the transcode workers:
|
||||
|
||||
| Variable | Required | Description |
|
||||
| ---------------------------------------- | --------------- | ------------------------------------------------------------------------------------- |
|
||||
| `TURNTABLE_RENDERER_URL` | Yes (to enable) | Base URL of the turntable renderer service, e.g. `http://turntable-renderer:3000`. |
|
||||
| `TURNTABLE_RENDERER_BASIC_AUTH_USERNAME` | No | Username when Basic Authentication is enabled (alias: `TURNTABLE_RENDERER_USERNAME`). |
|
||||
| `TURNTABLE_RENDERER_BASIC_AUTH_PASSWORD` | No | Password when Basic Authentication is enabled (alias: `TURNTABLE_RENDERER_PASSWORD`). |
|
||||
|
||||
### Configuration Priority & Read-Only Mode
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Environment variables take highest priority.** If `TURNTABLE_RENDERER_URL` is set in your environment, Shumai will always use the environment variable configuration.
|
||||
>
|
||||
> When configured via environment variables, the settings in **Team Settings → Integrations** will display a **Configured via environment variables (Read-only)** badge, and the input fields will be locked to prevent accidental changes or configuration drift.
|
||||
|
||||
---
|
||||
|
||||
## Deploying Turntable Renderer
|
||||
|
||||
### Docker Quickstart
|
||||
|
||||
The easiest way to run `turntable-renderer` is using the prebuilt Docker image:
|
||||
|
||||
```bash
|
||||
docker run -d --rm \
|
||||
--name turntable-renderer \
|
||||
-p 3003:3000 \
|
||||
-e TURNTABLE_RENDERER_API_BASIC_AUTH_USERNAME=admin \
|
||||
-e TURNTABLE_RENDERER_API_BASIC_AUTH_PASSWORD=your_password \
|
||||
shumaione/turntable-renderer:latest
|
||||
```
|
||||
|
||||
> **Note**: Host port `3003` maps to internal container port `3000`.
|
||||
|
||||
### Docker Compose Setup
|
||||
|
||||
Add a `turntable-renderer` service to your `docker-compose.yaml`:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
turntable-renderer:
|
||||
image: shumaione/turntable-renderer:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- '3003:3000'
|
||||
environment:
|
||||
TURNTABLE_RENDERER_API_BASIC_AUTH_USERNAME: admin
|
||||
TURNTABLE_RENDERER_API_BASIC_AUTH_PASSWORD: your_password
|
||||
```
|
||||
|
||||
Then point Shumai at it:
|
||||
|
||||
```bash
|
||||
TURNTABLE_RENDERER_URL=http://localhost:3003
|
||||
TURNTABLE_RENDERER_BASIC_AUTH_USERNAME=admin
|
||||
TURNTABLE_RENDERER_BASIC_AUTH_PASSWORD=your_password
|
||||
```
|
||||
|
||||
_(If both Shumai and turntable-renderer run within the same Docker Compose network, use `http://turntable-renderer:3000` as the URL)._
|
||||
|
||||
### GPU Acceleration
|
||||
|
||||
A dedicated GPU is optional — turntable-renderer's CPU rendering pipeline is fast enough for modest hardware. When a GPU is passed into the container, it is automatically detected and leveraged.
|
||||
|
||||
#### Intel / AMD (Open Source Drivers)
|
||||
|
||||
Pass the DRI render node:
|
||||
|
||||
```bash
|
||||
docker run -d --rm \
|
||||
--name turntable-renderer \
|
||||
--shm-size=1gb \
|
||||
-p 3003:3000 \
|
||||
--device /dev/dri \
|
||||
-e TURNTABLE_RENDERER_API_BASIC_AUTH_USERNAME=admin \
|
||||
-e TURNTABLE_RENDERER_API_BASIC_AUTH_PASSWORD=your_password \
|
||||
shumaione/turntable-renderer:latest
|
||||
```
|
||||
|
||||
#### NVIDIA (Proprietary Drivers)
|
||||
|
||||
Requires host driver 580+ and the NVIDIA Container Toolkit (`--runtime nvidia`):
|
||||
|
||||
```bash
|
||||
docker run -d --rm \
|
||||
--name turntable-renderer \
|
||||
--shm-size=1gb \
|
||||
-p 3003:3000 \
|
||||
--runtime nvidia \
|
||||
--gpus all \
|
||||
--device /dev/nvidia-modeset \
|
||||
-e TURNTABLE_RENDERER_API_BASIC_AUTH_USERNAME=admin \
|
||||
-e TURNTABLE_RENDERER_API_BASIC_AUTH_PASSWORD=your_password \
|
||||
shumaione/turntable-renderer:latest
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Behavior Without Turntable Renderer
|
||||
|
||||
If turntable-renderer is not configured or unreachable:
|
||||
|
||||
- The 3D model upload completes successfully with status `processed`.
|
||||
- No turntable video proxy, sprite, or poster is generated.
|
||||
- The original raw 3D file is safely preserved and can be downloaded by team members.
|
||||
@@ -0,0 +1,131 @@
|
||||
---
|
||||
title: Embedding Server Setup (EmbeddingGemma 2)
|
||||
description: Run local, offline multimodal vector embeddings for semantic search using shumaiOne/embedding-server.
|
||||
---
|
||||
|
||||
Shumai supports local, self-hosted multimodal vector embeddings for semantic search through [shumaiOne/embedding-server](https://github.com/shumaiOne/embedding-server) — a standalone Docker service powered by Google AI Edge's **LiteRT-LM** and **EmbeddingGemma 2 740M**.
|
||||
|
||||
The embedding server provides a unified 768-dimensional vector space across text, images, and video frames with an OpenAI-compatible HTTP API (`/v1/embeddings`). Because the model weights are pre-baked directly into the Docker image, the server operates completely offline and air-gapped without requiring external network access or third-party cloud API keys.
|
||||
|
||||
---
|
||||
|
||||
## Model Architecture & Specs
|
||||
|
||||
| Property | Value |
|
||||
| :------------------------ | :------------------------------------------------------- |
|
||||
| **Model** | `EmbeddingGemma 2 740M` (`google/embeddinggemma-2-740m`) |
|
||||
| **Runtime** | Google AI Edge LiteRT-LM (Python 3.14 Debian slim) |
|
||||
| **Vector Dimensionality** | 768 dimensions (multimodal: text, image, and video) |
|
||||
| **Context Window** | 2,048 tokens |
|
||||
| **Network Mode** | Completely offline capable (`--network none`) |
|
||||
| **API Standard** | OpenAI-compatible `/v1/embeddings` |
|
||||
|
||||
---
|
||||
|
||||
## Video Processing & Sampling
|
||||
|
||||
For video assets, Shumai breaks videos into standardized segments and extracts keyframes:
|
||||
|
||||
- **Chunking**: Videos are divided into **60-second chunks** with a **5-second overlap** at segment boundaries to preserve narrative and visual context.
|
||||
- **Frame Extraction Rate**: Shumai extracts **15 frames per minute** from each chunk.
|
||||
- **Adaptive Frame Bounding**:
|
||||
- For standard aspect ratios (1:1, 4:3, 3:4, 16:9, 9:16, 21:9), Shumai extracts 15 frames, which produces ~1,982 prompt tokens and stays safely within the 2,048-token context window.
|
||||
- For non-standard or custom aspect ratios (e.g. 5:3, 1.4:1), Shumai automatically samples 14 frames (~1,990 tokens) to mathematically guarantee that prompt length never exceeds 2,048 tokens, preventing out-of-memory errors on the embedding server.
|
||||
|
||||
---
|
||||
|
||||
## Important: Changing Embedding Providers Breaks Existing Embeddings
|
||||
|
||||
> [!WARNING]
|
||||
> **Vector spaces from different embedding models are completely incompatible.**
|
||||
>
|
||||
> If you change your embedding provider (for example, switching from Google Gemini's 1536-dimensional embeddings to EmbeddingGemma's 768-dimensional embeddings, or vice versa), **all existing vector embeddings in your database become invalid**.
|
||||
>
|
||||
> To protect production databases from accidental corruption and search failures, **Shumai intentionally does not provide a WebUI toggle to change the embedding provider**. The provider can only be configured by an administrator via environment variables (`EMBEDDING_PROVIDER` and `GEMMA_EMBEDDING_OPENAI_BASE_URL`).
|
||||
|
||||
---
|
||||
|
||||
## Regenerating Embeddings via CLI
|
||||
|
||||
When you do switch embedding providers or configure the embedding server on an existing Shumai deployment, you can regenerate embeddings for all existing media files using the administrative CLI command:
|
||||
|
||||
### Inside Docker
|
||||
|
||||
```bash
|
||||
docker exec -it <shumai-container-name> shumai regenerate-embeddings
|
||||
```
|
||||
|
||||
### Local / Bare Metal
|
||||
|
||||
```bash
|
||||
shumai regenerate-embeddings
|
||||
```
|
||||
|
||||
_(Alternatively, `bun apps/web/src/index.ts regenerate-embeddings`)_
|
||||
|
||||
### What the Command Does
|
||||
|
||||
1. **Scans Teams**: Checks all teams in your workspace. If a team does not have an active (enabled) embedding agent, it logs a warning and safely skips that team.
|
||||
2. **Clears Existing Embeddings**: Purges all previous embeddings and cancels any pending/in-progress embedding tasks for eligible assets to ensure a clean transition to the new provider.
|
||||
3. **Enqueues Fresh Tasks**: Queues new background `ai_embedding` tasks for all eligible assets (images, videos, and 3D assets with rendered previews) across active teams.
|
||||
4. **Background Execution**: Background workers process the queued tasks according to your configured concurrency limits.
|
||||
|
||||
---
|
||||
|
||||
## Deployment & Configuration
|
||||
|
||||
### Docker Quickstart
|
||||
|
||||
To run the standalone embedding server container:
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name shumai_embedding \
|
||||
-p 9379:9379 \
|
||||
shumaione/embedding-server:latest
|
||||
```
|
||||
|
||||
### Docker Compose Setup
|
||||
|
||||
In Docker Compose environments, add `embedding-server` to your `docker-compose.yaml` and configure Shumai to connect to it over Docker's internal network:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
embedding-server:
|
||||
image: shumaione/embedding-server:latest
|
||||
container_name: shumai_embedding
|
||||
restart: unless-stopped
|
||||
|
||||
shumai:
|
||||
image: shumaione/shumai:latest
|
||||
environment:
|
||||
# Embedding Server Configuration
|
||||
EMBEDDING_PROVIDER: gemma-embedding-2-openai
|
||||
GEMMA_EMBEDDING_OPENAI_BASE_URL: http://embedding-server:9379/v1
|
||||
ENABLE_EMBEDDING_AGENT: 'true'
|
||||
CONCURRENCY_AGENT: 1
|
||||
depends_on:
|
||||
- embedding-server
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Since Shumai communicates with `embedding-server` over Docker's internal network, `embedding-server` does not need to expose any ports to the host machine.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Required | Description | Default |
|
||||
| :-------------------------------- | :---------- | :---------------------------------------------------------------------------------------------------------------------------- | :------------------------- |
|
||||
| `EMBEDDING_PROVIDER` | Yes | Set to `gemma-embedding-2-openai` to use the local LiteRT-LM server. | `gemini-embedding-2` |
|
||||
| `GEMMA_EMBEDDING_OPENAI_BASE_URL` | Yes | HTTP base URL of the embedding server (e.g. `http://embedding-server:9379/v1`). | `http://localhost:9379/v1` |
|
||||
| `ENABLE_EMBEDDING_AGENT` | No | Automatically provisions and enables the system embedding agent for all teams on startup. | `false` |
|
||||
| `CONCURRENCY_AGENT` | Recommended | Maximum concurrent AI tasks executed by workers. **Set to `1`** when running `embedding-server` to prevent high memory usage. | `5` |
|
||||
|
||||
---
|
||||
|
||||
## Hardware & Resource Requirements
|
||||
|
||||
- **Memory (RAM)**:
|
||||
- We recommend at least **4 GB+ of dedicated system RAM** for the `embedding-server` container.
|
||||
- Overall host system memory should be **4 GB+** when running the Lite+Embedding compose stack, and **8 GB+** for the full stack.
|
||||
- **Concurrency**:
|
||||
- Always set `CONCURRENCY_AGENT=1` in your Shumai configuration when using `embedding-server`. Parallel multimodal embedding generations consume significant memory and can cause Docker containers to be terminated by the OS Out-Of-Memory (OOM) killer on low-resource machines.
|
||||
@@ -32,7 +32,22 @@ When Gotenberg is configured, Shumai automatically generates a PDF proxy for the
|
||||
|
||||
## Configuration
|
||||
|
||||
Gotenberg support is opt-in and controlled by the following environment variables on the machine running the transcode workers:
|
||||
Shumai supports configuring Gotenberg either through the **WebUI** or via **Environment Variables**.
|
||||
|
||||
### 1. WebUI (Team Settings → Integrations)
|
||||
|
||||
Team owners can configure Gotenberg directly from the web interface:
|
||||
|
||||
1. Navigate to **Team Settings** from the sidebar.
|
||||
2. Select the **Integrations** tab.
|
||||
3. Under **Gotenberg Document Converter**, enter the **Server URL** (e.g. `http://gotenberg:3000` or `http://localhost:3000`).
|
||||
4. Enter the **Username** and **Password** if Basic Authentication is enabled on the service.
|
||||
5. Click **Test Connection** to test communication with Gotenberg and verify its health.
|
||||
6. Click **Save Changes**.
|
||||
|
||||
### 2. Environment Variables
|
||||
|
||||
Gotenberg can also be configured globally across the system using environment variables on the machine running Shumai or the transcode workers:
|
||||
|
||||
| Variable | Required | Description |
|
||||
| ------------------------------- | --------------- | ---------------------------------------------------------------- |
|
||||
@@ -40,6 +55,15 @@ Gotenberg support is opt-in and controlled by the following environment variable
|
||||
| `GOTENBERG_BASIC_AUTH_USERNAME` | No | Username when Basic Authentication is enabled on Gotenberg. |
|
||||
| `GOTENBERG_BASIC_AUTH_PASSWORD` | No | Password when Basic Authentication is enabled on Gotenberg. |
|
||||
|
||||
### Configuration Priority & Read-Only Mode
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Environment variables take highest priority.** If `GOTENBERG_URL` is set in your environment, Shumai will always use the environment variable configuration.
|
||||
>
|
||||
> When configured via environment variables, the settings in **Team Settings → Integrations** will display a **Configured via environment variables (Read-only)** badge, and the input fields will be locked to prevent accidental changes or configuration drift.
|
||||
|
||||
### Deploying Gotenberg
|
||||
|
||||
### Docker Quickstart
|
||||
|
||||
You can start a Gotenberg container with a single command:
|
||||
|
||||
@@ -12,11 +12,10 @@ Traditional search relies on matching exact characters (e.g., searching "dog" wi
|
||||
Semantic search translates text queries and file contents into conceptual profiles. Similar ideas and visuals are mapped near each other.
|
||||
|
||||
1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile.
|
||||
- **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing.
|
||||
- **Chunking**: For long videos, Shumai breaks them into chunks to ensure high-quality indexing. By default, chunks are **60 seconds** long with a **5-second overlap** to ensure no context is lost at the boundaries.
|
||||
- **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames.
|
||||
- **Optimization**: To optimize processing, Shumai automatically transcodes each chunk to **1 FPS** and **removes the audio track** before sending it to the embedding agent.
|
||||
- **Configuration**: Administrators can customize these settings using the `EMBEDDING_CHUNK_DURATION` and `EMBEDDING_CHUNK_OVERLAP` environment variables.
|
||||
- **Multimodal Embedding Providers**: Shumai supports two embedding providers:
|
||||
- **EmbeddingGemma 2 (Local)**: Powered by `shumaiOne/embedding-server` with 768-dimensional multimodal vectors running completely offline. Videos are sampled at **15 frames per minute** (14 frames for non-standard aspect ratios) per 60-second chunk. See the [Embedding Server Setup Guide](./embedding-server) for full details.
|
||||
- **Gemini Embedding 2 (Cloud)**: Powered by Google's `gemini-embedding-2` API with 1536-dimensional vectors. Videos are chunked into 60-second segments (sampled up to 32 frames per chunk at 1 fps with audio stripped).
|
||||
- **Chunking Configuration**: Administrators can customize chunk duration and overlap using the `EMBEDDING_CHUNK_DURATION` (default: 60s) and `EMBEDDING_CHUNK_OVERLAP` (default: 5s) environment variables.
|
||||
2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets.
|
||||
3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`.
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
@@ -9,7 +9,7 @@
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.74",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@earendil-works/pi-ai": "0.82.1",
|
||||
"@google/genai": "2.19.0",
|
||||
"@google/genai": "2.28.0",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
@@ -19,6 +19,7 @@
|
||||
"@temporalio/workflow": "1.23.0",
|
||||
"adm-zip": "0.6.0",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"sharp": "0.35.4",
|
||||
"typebox": "1.3.22",
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.5.4"
|
||||
|
||||
@@ -662,6 +662,23 @@ describe('Agent Activities', () => {
|
||||
const pdfPrompt = mockHarness.prompt.mock.calls[1]?.[0] ?? ''
|
||||
expect(pdfPrompt).toContain('Document pages: 50')
|
||||
expect(pdfPrompt).not.toContain('Video duration')
|
||||
|
||||
// For a 3D asset, should produce 3D Model recommendation
|
||||
await autofillAiActivity({
|
||||
teamId: 't1',
|
||||
assetId: 'model-3',
|
||||
assetName: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
images: [],
|
||||
fields: [{ id: 'f1', config: { name: 'F1', type: 'text' } }],
|
||||
context,
|
||||
})
|
||||
|
||||
const modelPrompt = mockHarness.prompt.mock.calls[2]?.[0] ?? ''
|
||||
expect(modelPrompt).toContain('3D Model: This asset has an orthogonal 4-view composite image')
|
||||
expect(modelPrompt).toContain(
|
||||
'Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1170,6 +1187,43 @@ describe('Agent Database Activities Integration', () => {
|
||||
expect((rootEntry as any)?.details?.thread).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should set media position to degree for 3D asset comments', async () => {
|
||||
const asset3d = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo,
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
const comment3d = await prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: asset3d.id,
|
||||
message: 'Nice angle',
|
||||
creatorId: user.id,
|
||||
second: 0.5, // 45°
|
||||
},
|
||||
})
|
||||
|
||||
const sessionId = await initializeAgentSessionActivity({
|
||||
teamId: team.id,
|
||||
agentId: 'test-agent-id',
|
||||
userCommentId: comment3d.id,
|
||||
userId: user.id,
|
||||
})
|
||||
|
||||
const storage = new DatabaseSessionStorage(sessionId)
|
||||
const pathEntries = await storage.getPathToRoot(comment3d.id)
|
||||
const entry = pathEntries.find((e) => e.id === comment3d.id)
|
||||
|
||||
expect(entry).toBeDefined()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- test entry position parsing
|
||||
expect((entry as any)?.details?.position).toEqual({ type: 'degree', degree: 45 })
|
||||
})
|
||||
|
||||
it('should return user name and role for team member', async () => {
|
||||
const info = await getUserTeamInfoActivity({
|
||||
userId: user.id,
|
||||
@@ -2105,5 +2159,83 @@ describe('Agent Database Activities Integration', () => {
|
||||
expect(systemPromptArg).toContain('<thread id="..." reply_count="..." />')
|
||||
expect(systemPromptArg).toContain('read_thread')
|
||||
})
|
||||
|
||||
it('agentChatActivity injects # 3D Model Inspection into systemPrompt', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User_3d_Prompt', email: `prompt-3d-${Date.now()}@example.com` },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team_3d_Prompt_' + Date.now() },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project_3d_Prompt', teamId: team.id },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo,
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
const comment = await prisma.assetComment.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
creatorId: user.id,
|
||||
message: 'Check this 3d model',
|
||||
},
|
||||
})
|
||||
|
||||
const mockSession = {
|
||||
id: 'mock-session-id',
|
||||
subscribe: vi.fn(),
|
||||
getEntries: vi.fn().mockResolvedValue([]),
|
||||
getStorage: vi.fn().mockReturnValue({ sessionId: 'mock-session-id' }),
|
||||
}
|
||||
const mockHarness = {
|
||||
subscribe: vi.fn(),
|
||||
prompt: vi.fn().mockResolvedValue({
|
||||
content: [{ type: 'text', text: 'Response' }],
|
||||
usage: { input: 10, output: 20 },
|
||||
}),
|
||||
}
|
||||
|
||||
vi.mocked(piAgent.createAgentSession).mockResolvedValue({
|
||||
session: mockSession as unknown as Session<DatabaseSessionMetadata>,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock AgentHarness instance for activity test
|
||||
harness: mockHarness as unknown as AgentHarness<any, any, any, any>,
|
||||
})
|
||||
|
||||
const context = {
|
||||
agent: { id: 'agent-1', provider: { name: 'google' }, modelRef: { modelId: 'gemini' } },
|
||||
dbProviders: [],
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
} as unknown as AgentExecutionContext
|
||||
|
||||
await agentChatActivity({
|
||||
teamId: team.id,
|
||||
agentId: 'agent-1',
|
||||
message: 'Hello',
|
||||
imageUrls: [],
|
||||
projectId: project.id,
|
||||
folderId: asset.id,
|
||||
assetId: asset.id,
|
||||
sessionId: 'mock-session-id',
|
||||
userCommentId: comment.id,
|
||||
userId: user.id,
|
||||
context,
|
||||
})
|
||||
|
||||
const lastCall = vi.mocked(piAgent.createAgentSession).mock.calls.at(-1)
|
||||
const systemPromptArg = lastCall?.[0].systemPrompt as string
|
||||
expect(systemPromptArg).toContain('# 3D Model Inspection')
|
||||
expect(systemPromptArg).toContain('media_type="3d"')
|
||||
expect(systemPromptArg).toContain('config3d')
|
||||
expect(systemPromptArg).toContain('<position type="degree" degree="..." />')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
import { aiUsageService } from '@shumai/core/src/ai-usage/ai-usage'
|
||||
import { quotaService, QuotaExceededError } from '@shumai/core/src/quota/quota-service'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
import { secondToDegree } from '@shumai/core/src/utils/model3d'
|
||||
import {
|
||||
UpdateAssetMetadataRequest,
|
||||
type ShumaiMessageContext,
|
||||
@@ -179,6 +180,8 @@ User messages may contain a <context> block detailing the user, active asset loc
|
||||
systemPrompt += `\n\n# Comment Threads\nWhen operating in comment mode, previous top-level messages in the conversation history may contain a <thread id="..." reply_count="..." /> tag indicating an earlier discussion thread with replies. If a user's question refers to or depends on earlier comments or discussions, you can use the 'read_thread' tool with the thread ID to inspect all replies in that thread.`
|
||||
}
|
||||
|
||||
systemPrompt += `\n\n# 3D Model Inspection\nWhen interacting with 3D model assets (media_type="3d"), you can visually inspect the asset using 'read_asset'. Calling 'read_asset' with config3d: null (or config3d: {}) returns an orthogonal 4-view composite image (Front at top-left, Right side at top-right, Back at bottom-left, Left side at bottom-right). If a <position type="degree" degree="..." /> tag is present or a specific angle is discussed, pass 'config3d: { degree: ... }' (integer 0, 15, 30, ..., 345) to inspect that view angle. If an <annotation id="..." /> tag is present on a 3D asset, pass 'annotationId' to inspect the angle with the user's visual markup drawings overlaid.`
|
||||
|
||||
if (agent.type === 'autofill') {
|
||||
systemPrompt += `\n\n# Autofill Instructions\nWhen performing metadata autofill, inspect the asset content using the 'read_asset' tool. If 'read_asset' fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.`
|
||||
}
|
||||
@@ -564,6 +567,7 @@ export async function autofillAiActivity(params: AutofillAiParams) {
|
||||
const proxyType = getProxyType(params.mediaType, params.assetName)
|
||||
const isVideoOrAudio = proxyType === 'video' || proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
const is3d = proxyType === '3d'
|
||||
|
||||
if (params.duration !== undefined && params.duration > 0 && (!proxyType || isVideoOrAudio)) {
|
||||
promptLines.push(
|
||||
@@ -575,11 +579,16 @@ export async function autofillAiActivity(params: AutofillAiParams) {
|
||||
`- Document pages: ${params.pageCount}. Recommended inspection: call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ${Math.min(params.pageCount, 20)} } or mode: "text".`,
|
||||
)
|
||||
}
|
||||
if (is3d) {
|
||||
promptLines.push(
|
||||
'- 3D Model: This asset has an orthogonal 4-view composite image (2x2 grid showing Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right). Call read_asset to inspect the 4 views.',
|
||||
)
|
||||
}
|
||||
|
||||
promptLines.push(
|
||||
'',
|
||||
'Instructions:',
|
||||
`1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type).`,
|
||||
`1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type; for 3D assets, call read_asset directly to inspect the 4-view composite image).`,
|
||||
'2. After analyzing the retrieved content, call the "autofill_metadata" tool to provide the extracted metadata values.',
|
||||
'3. If "read_asset" fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.',
|
||||
)
|
||||
@@ -834,7 +843,9 @@ export async function initializeAgentSessionActivity(params: {
|
||||
|
||||
let position: ShumaiMediaPosition | undefined = undefined
|
||||
if (c.second !== null && c.second !== undefined) {
|
||||
if (proxyType === 'pdf') {
|
||||
if (proxyType === '3d') {
|
||||
position = { type: 'degree', degree: secondToDegree(c.second) }
|
||||
} else if (proxyType === 'pdf') {
|
||||
position = { type: 'page', page: Math.round(c.second) }
|
||||
} else {
|
||||
position = { type: 'time', seconds: c.second }
|
||||
@@ -852,7 +863,14 @@ export async function initializeAgentSessionActivity(params: {
|
||||
id: att.asset.id,
|
||||
name: att.asset.name,
|
||||
type: att.asset.type,
|
||||
mediaType: attProxyType as 'image' | 'video' | 'pdf' | 'audio' | 'other' | undefined,
|
||||
mediaType: attProxyType as
|
||||
| 'image'
|
||||
| 'video'
|
||||
| 'pdf'
|
||||
| 'audio'
|
||||
| '3d'
|
||||
| 'other'
|
||||
| undefined,
|
||||
mimeType: att.asset.mediaType || undefined,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import {
|
||||
generateTextEmbeddingActivity,
|
||||
extractAiMetadataActivity,
|
||||
getEmbeddingContextActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
deleteAssetEmbeddingsActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
generateEmbeddingActivity,
|
||||
getGemmaEmbeddingTargetFrameCount,
|
||||
} from './ai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { prisma, AssetType, AssetStatus } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import sharp from 'sharp'
|
||||
|
||||
vi.mock('fs', async (importOriginal) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- importOriginal returns the raw fs module, typed dynamically as any here
|
||||
@@ -54,7 +56,6 @@ vi.mock('child_process', () => {
|
||||
const execFileMock: any = vi.fn((file: any, args: any, cb: any) => {
|
||||
cb(null, '120.0\n', '')
|
||||
})
|
||||
// Setup custom promisify behavior to return { stdout, stderr }
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mock functions dynamically return varying types based on test command calls
|
||||
execFileMock[Symbol.for('nodejs.util.promisify.custom')] = vi.fn(async (file: any) => {
|
||||
if (file === 'ffprobe') {
|
||||
@@ -90,6 +91,14 @@ describe('AI Activities Unit Tests', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
process.env.GEMINI_API_KEY = 'test-key'
|
||||
delete process.env.EMBEDDING_PROVIDER
|
||||
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
delete process.env.EMBEDDING_PROVIDER
|
||||
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
|
||||
})
|
||||
|
||||
it('should generate image embedding', async () => {
|
||||
@@ -126,33 +135,6 @@ describe('AI Activities Unit Tests', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should resolve video input via resolveInput and avoid downloadToFile in generateEmbeddingActivity', async () => {
|
||||
vi.mocked(s3Service.downloadToFile).mockClear()
|
||||
vi.mocked(s3Service.resolveInput).mockClear()
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue(
|
||||
'https://mock-storage/files/asset-123/video.mp4',
|
||||
)
|
||||
|
||||
const res = await generateEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetId: 'asset-123',
|
||||
context: {
|
||||
agent: {},
|
||||
asset: {
|
||||
mediaType: 'video/mp4',
|
||||
name: 'sample.mp4',
|
||||
storageKey: { key: 'files/asset-123/video.mp4' },
|
||||
media: { proxyType: 'video' },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(s3Service.resolveInput).toHaveBeenCalledWith('shumai', 'files/asset-123/video.mp4')
|
||||
expect(s3Service.downloadToFile).not.toHaveBeenCalled()
|
||||
expect(s3Service.getObject).not.toHaveBeenCalledWith('shumai', 'files/asset-123/video.mp4')
|
||||
expect(res.embeddings.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should generate text embedding successfully', async () => {
|
||||
const res = await generateTextEmbeddingActivity({
|
||||
text: 'Search query',
|
||||
@@ -172,6 +154,356 @@ describe('AI Activities Unit Tests', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate text embedding successfully with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.1) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 15 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const res = await generateTextEmbeddingActivity({
|
||||
text: 'Search query',
|
||||
teamId: 't1',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(res.usage.model).toBe('embeddinggemma-2')
|
||||
expect(res.usage.inputTokens).toBe(15)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input: 'task: search query | text: Search query',
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate image embedding successfully with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.2) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('test-image'),
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateImageEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetKey: 'img.png',
|
||||
mediaType: 'image/png',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(res.usage.model).toBe('embeddinggemma-2')
|
||||
expect(res.usage.inputTokens).toBe(140)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input: [
|
||||
{
|
||||
type: 'image_url',
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from('test-image').toString('base64')}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should respect custom GEMMA_EMBEDDING_OPENAI_BASE_URL with or without /v1 suffix', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.2) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 10 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
// Test with /v1
|
||||
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379/v1'
|
||||
await generateTextEmbeddingActivity({ text: 'test query', teamId: 't1' })
|
||||
expect(mockFetch).toHaveBeenLastCalledWith(
|
||||
'http://custom-host:9379/v1/embeddings',
|
||||
expect.anything(),
|
||||
)
|
||||
|
||||
// Test without /v1
|
||||
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379'
|
||||
await generateTextEmbeddingActivity({ text: 'test query', teamId: 't1' })
|
||||
expect(mockFetch).toHaveBeenLastCalledWith(
|
||||
'http://custom-host:9379/v1/embeddings',
|
||||
expect.anything(),
|
||||
)
|
||||
})
|
||||
|
||||
it('should convert webp image to jpeg with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.2) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const webpBuffer = await sharp({
|
||||
create: { width: 10, height: 10, channels: 3, background: { r: 255, g: 0, b: 0 } },
|
||||
})
|
||||
.webp()
|
||||
.toBuffer()
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: webpBuffer,
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateImageEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetKey: 'img.webp',
|
||||
mediaType: 'image/webp',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: expect.stringContaining('data:image/jpeg;base64,'),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate video chunk embedding from tar archive with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.3) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const sampleArchive = new Bun.Archive({
|
||||
'frame_0001.jpg': Buffer.from('frame-1'),
|
||||
})
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(res.usage.model).toBe('embeddinggemma-2')
|
||||
expect(mockFetch).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
describe('getGemmaEmbeddingTargetFrameCount', () => {
|
||||
it('should return 15 for standard aspect ratios', () => {
|
||||
// 16:9
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1920, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1280, 720)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(854, 480)).toBe(15)
|
||||
// 9:16
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1080, 1920)).toBe(15)
|
||||
// 1:1
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1080, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(480, 480)).toBe(15)
|
||||
// 4:3
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1440, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(640, 480)).toBe(15)
|
||||
// 3:4
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1080, 1440)).toBe(15)
|
||||
// 21:9
|
||||
expect(getGemmaEmbeddingTargetFrameCount(2560, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(3440, 1440)).toBe(15)
|
||||
})
|
||||
|
||||
it('should return 14 for non-standard aspect ratios and missing/invalid dimensions', () => {
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1000, 600)).toBe(14) // 5:3
|
||||
expect(getGemmaEmbeddingTargetFrameCount(700, 500)).toBe(14) // 1.4:1
|
||||
expect(getGemmaEmbeddingTargetFrameCount(854, 672)).toBe(14) // ~1.27:1
|
||||
expect(getGemmaEmbeddingTargetFrameCount(undefined, undefined)).toBe(14)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(0, 0)).toBe(14)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(-100, 100)).toBe(14)
|
||||
})
|
||||
})
|
||||
|
||||
it('should sample uniform 15 frames when chunk archive contains standard 16:9 frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.3) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const sampleJpg = await sharp({
|
||||
create: {
|
||||
width: 160,
|
||||
height: 90,
|
||||
channels: 3,
|
||||
background: { r: 0, g: 0, b: 0 },
|
||||
},
|
||||
})
|
||||
.jpeg()
|
||||
.toBuffer()
|
||||
|
||||
const frames: Record<string, Buffer> = {}
|
||||
for (let i = 1; i <= 30; i++) {
|
||||
frames[`frame_${String(i).padStart(4, '0')}.jpg`] = sampleJpg
|
||||
}
|
||||
const sampleArchive = new Bun.Archive(frames)
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
|
||||
expect(requestBody.input).toHaveLength(15)
|
||||
})
|
||||
|
||||
it('should fallback to 14 frames when chunk archive contains non-standard or unparseable frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.3) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const frames: Record<string, Buffer> = {}
|
||||
for (let i = 1; i <= 30; i++) {
|
||||
frames[`frame_${String(i).padStart(4, '0')}.jpg`] = Buffer.from(`frame-${i}`)
|
||||
}
|
||||
const sampleArchive = new Bun.Archive(frames)
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
|
||||
expect(requestBody.input).toHaveLength(14)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when chunk archive contains no valid frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const sampleArchive = new Bun.Archive({
|
||||
'info.txt': Buffer.from('not an image'),
|
||||
})
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(
|
||||
generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
}),
|
||||
).rejects.toThrow('No extracted frames found in chunk archive')
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when chunk archive data is corrupted', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('corrupted non-tar bytes'),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(
|
||||
generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
}),
|
||||
).rejects.toThrow('Failed to read chunk archive')
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when an extracted frame is empty (0 bytes)', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const sampleArchive = new Bun.Archive({
|
||||
'frame_0001.jpg': Buffer.alloc(0),
|
||||
})
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(
|
||||
generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
}),
|
||||
).rejects.toThrow('is empty (0 bytes)')
|
||||
})
|
||||
|
||||
describe('extractAiMetadataActivity', () => {
|
||||
it('should return existing images if they are already in S3 storage', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mock returned object cast as any due to partial implementation
|
||||
@@ -331,6 +663,18 @@ describe('AI Database Activities Integration', () => {
|
||||
expect(ctx.agent.id).toBe(user.id)
|
||||
expect(ctx.asset.id).toBe(asset.id)
|
||||
expect(ctx.asset.mediaType).toBe('video/mp4')
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
try {
|
||||
await expect(
|
||||
getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
}),
|
||||
).rejects.toThrow('embedding feature is disabled or agent not found')
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
it('should parse valid positive chunk duration and fallback on invalid or negative ones', async () => {
|
||||
@@ -389,6 +733,45 @@ describe('AI Database Activities Integration', () => {
|
||||
delete process.env.EMBEDDING_CHUNK_DURATION
|
||||
delete process.env.EMBEDDING_CHUNK_OVERLAP
|
||||
})
|
||||
|
||||
it('should return existing embeddings for the asset', async () => {
|
||||
// Ensure agent exists
|
||||
await prisma.agent.upsert({
|
||||
where: { id: user.id },
|
||||
update: {},
|
||||
create: {
|
||||
id: user.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: 'openai', model: 'gpt-4' },
|
||||
},
|
||||
})
|
||||
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings: [
|
||||
{ embedding: Array(1536).fill(0.01), startTime: 0.0, endTime: 60.0 },
|
||||
{ embedding: Array(768).fill(0.02), startTime: 55.0, endTime: 115.0 },
|
||||
],
|
||||
})
|
||||
|
||||
const ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
|
||||
expect(ctx.existingEmbeddings).toHaveLength(2)
|
||||
const chunk1 = ctx.existingEmbeddings.find((e) => e.startTime === 0)
|
||||
expect(chunk1).toBeDefined()
|
||||
expect(chunk1?.has1536).toBe(true)
|
||||
expect(chunk1?.has768).toBe(false)
|
||||
|
||||
const chunk2 = ctx.existingEmbeddings.find((e) => e.startTime === 55)
|
||||
expect(chunk2).toBeDefined()
|
||||
expect(chunk2?.has1536).toBe(false)
|
||||
expect(chunk2?.has768).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('saveAssetEmbeddingsActivity', () => {
|
||||
@@ -423,5 +806,85 @@ describe('AI Database Activities Integration', () => {
|
||||
expect(dbEmbeddings[0].endTime).toBe(60.0)
|
||||
expect(dbEmbeddings[1].startTime).toBeNull()
|
||||
})
|
||||
|
||||
it('should save 768-dim embeddings to embedding_768 column in database', async () => {
|
||||
const embeddings = [
|
||||
{
|
||||
embedding: Array(768).fill(0.01),
|
||||
startTime: 0.0,
|
||||
endTime: 60.0,
|
||||
},
|
||||
{
|
||||
embedding: Array(768).fill(0.02),
|
||||
},
|
||||
]
|
||||
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings,
|
||||
})
|
||||
|
||||
// Query raw embeddings from DB
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- db query output is a raw postgres result array typed as any
|
||||
const dbEmbeddings: any[] = await prisma.$queryRaw`
|
||||
SELECT id, asset_id as "assetId", (embedding_768 IS NOT NULL) as "has768", (embedding IS NOT NULL) as "has1536"
|
||||
FROM asset_embeddings
|
||||
WHERE asset_id = ${asset.id}
|
||||
`
|
||||
|
||||
expect(dbEmbeddings.length).toBe(2)
|
||||
expect(dbEmbeddings[0].has768).toBe(true)
|
||||
expect(dbEmbeddings[0].has1536).toBe(false)
|
||||
expect(dbEmbeddings[1].has768).toBe(true)
|
||||
expect(dbEmbeddings[1].has1536).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteAssetEmbeddingsActivity', () => {
|
||||
it('should delete specific embeddings by id', async () => {
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings: [
|
||||
{ embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 },
|
||||
{ embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 },
|
||||
],
|
||||
})
|
||||
|
||||
const before = await prisma.assetEmbedding.findMany({
|
||||
where: { assetId: asset.id },
|
||||
})
|
||||
expect(before).toHaveLength(2)
|
||||
|
||||
const targetId = before[0].id
|
||||
await deleteAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
ids: [targetId],
|
||||
})
|
||||
|
||||
const after = await prisma.assetEmbedding.findMany({
|
||||
where: { assetId: asset.id },
|
||||
})
|
||||
expect(after).toHaveLength(1)
|
||||
expect(after[0].id).not.toBe(targetId)
|
||||
})
|
||||
|
||||
it('should delete all embeddings for an asset when ids is not provided', async () => {
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings: [
|
||||
{ embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 },
|
||||
{ embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 },
|
||||
],
|
||||
})
|
||||
|
||||
await deleteAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
})
|
||||
|
||||
const after = await prisma.assetEmbedding.findMany({
|
||||
where: { assetId: asset.id },
|
||||
})
|
||||
expect(after).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
+264
-171
@@ -1,17 +1,11 @@
|
||||
import { GoogleGenAI } from '@google/genai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { execFile } from 'child_process'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import { promisify } from 'util'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
import { ulid } from 'ulid'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
import sharp from 'sharp'
|
||||
|
||||
export interface Usage {
|
||||
inputTokens: number
|
||||
@@ -25,15 +19,6 @@ export interface GeneratedEmbedding {
|
||||
endTime?: number
|
||||
}
|
||||
|
||||
export interface GenerateEmbeddingParams {
|
||||
teamId: string
|
||||
assetId: string
|
||||
context: {
|
||||
agent: unknown
|
||||
asset: unknown
|
||||
}
|
||||
}
|
||||
|
||||
async function generateMultimodalEmbedding(
|
||||
ai: GoogleGenAI,
|
||||
buffer: Buffer,
|
||||
@@ -62,134 +47,66 @@ async function generateMultimodalEmbedding(
|
||||
return values
|
||||
}
|
||||
|
||||
export async function generateEmbeddingActivity(params: GenerateEmbeddingParams): Promise<{
|
||||
embeddings: GeneratedEmbedding[]
|
||||
interface OpenAiEmbeddingItem {
|
||||
type?: string
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url?: {
|
||||
url: string
|
||||
}
|
||||
}
|
||||
|
||||
async function callGemmaEmbeddingServer(input: string | OpenAiEmbeddingItem[]): Promise<{
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- context properties are fetched as JSON from the DB activity
|
||||
const { asset } = params.context as any
|
||||
const rawBaseUrl = process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL || 'http://localhost:9379/v1'
|
||||
|
||||
if (!asset.mediaType) {
|
||||
throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true })
|
||||
}
|
||||
const normalized = rawBaseUrl.replace(/\/+$/, '')
|
||||
const endpoint = normalized.endsWith('/embeddings')
|
||||
? normalized
|
||||
: normalized.endsWith('/v1')
|
||||
? `${normalized}/embeddings`
|
||||
: `${normalized}/v1/embeddings`
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input,
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
throw ApplicationFailure.create({
|
||||
message: 'GEMINI_API_KEY environment variable is not configured',
|
||||
nonRetryable: true,
|
||||
message: `Embedding server returned status ${response.status}: ${errorText}`,
|
||||
nonRetryable: false,
|
||||
})
|
||||
}
|
||||
|
||||
const resolvedApiKey = process.env[apiKey] || apiKey
|
||||
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
|
||||
|
||||
const usage: Usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- response json shape is dynamic from external endpoint
|
||||
const json: any = await response.json()
|
||||
const embedding = json.data?.[0]?.embedding
|
||||
if (!embedding || !Array.isArray(embedding)) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `unsupported media type for embeddings: ${asset.mediaType}`,
|
||||
nonRetryable: true,
|
||||
message: 'Invalid embedding returned from embedding server',
|
||||
nonRetryable: false,
|
||||
})
|
||||
}
|
||||
|
||||
const key = asset.storageKey?.key
|
||||
if (!key) {
|
||||
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
|
||||
}
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const results: GeneratedEmbedding[] = []
|
||||
|
||||
if (isImage) {
|
||||
const { buffer: data } = await s3Service.getObject(bucket, key)
|
||||
const embVec = await generateMultimodalEmbedding(ai, data, asset.mediaType)
|
||||
results.push({ embedding: embVec })
|
||||
} else if (isVideo) {
|
||||
const inputSource = await s3Service.resolveInput(bucket, key)
|
||||
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
|
||||
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
'-show_entries',
|
||||
'format=duration',
|
||||
'-of',
|
||||
'default=noprint_wrappers=1:nokey=1',
|
||||
inputSource,
|
||||
])
|
||||
const duration = parseFloat(stdout.trim())
|
||||
if (isNaN(duration)) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'failed to parse video duration',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const chunkSize = 60.0
|
||||
for (let start = 0.0; start < duration; start += chunkSize) {
|
||||
let end = start + chunkSize
|
||||
if (end > duration) end = duration
|
||||
|
||||
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'warning',
|
||||
...(isRemote
|
||||
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
|
||||
: []),
|
||||
'-ss',
|
||||
start.toString(),
|
||||
'-i',
|
||||
inputSource,
|
||||
'-map',
|
||||
'0:V:0',
|
||||
'-map',
|
||||
'0:a:0?',
|
||||
'-t',
|
||||
(end - start).toString(),
|
||||
'-c',
|
||||
'copy',
|
||||
chunkTmp,
|
||||
])
|
||||
|
||||
const chunkData = fs.readFileSync(chunkTmp)
|
||||
|
||||
try {
|
||||
const embVec = await generateMultimodalEmbedding(ai, chunkData, 'video/mp4')
|
||||
results.push({ embedding: embVec, startTime: start, endTime: end })
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} catch (e: any) {
|
||||
console.error(
|
||||
`failed to generate video embedding for chunk ${start}-${end}: ${e.message}`,
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
if (fs.existsSync(chunkTmp)) {
|
||||
fs.unlinkSync(chunkTmp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: results,
|
||||
usage,
|
||||
embedding,
|
||||
usage: {
|
||||
model: json.model || 'gemma-embedding-2-openai',
|
||||
inputTokens: json.usage?.prompt_tokens ?? 0,
|
||||
outputTokens: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
import { textEmbeddingService } from '@shumai/core/src/search/text-embedding'
|
||||
|
||||
export interface GenerateTextEmbeddingParams {
|
||||
text: string
|
||||
teamId: string
|
||||
@@ -199,44 +116,36 @@ export async function generateTextEmbeddingActivity(params: GenerateTextEmbeddin
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
if (provider !== 'gemma-embedding-2-openai' && !process.env.GEMINI_API_KEY) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'GEMINI_API_KEY environment variable is not configured',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const resolvedApiKey = process.env[apiKey] || apiKey
|
||||
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
|
||||
try {
|
||||
const result = await textEmbeddingService.generateEmbedding(params.text, {
|
||||
teamId: params.teamId,
|
||||
skipCache: true,
|
||||
})
|
||||
|
||||
const usage: Usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const response = await ai.models.embedContent({
|
||||
model: 'gemini-embedding-2',
|
||||
contents: [
|
||||
{
|
||||
parts: [{ text: params.text }],
|
||||
role: 'user',
|
||||
return {
|
||||
embedding: result.embedding,
|
||||
usage: {
|
||||
model: result.usage?.model || provider,
|
||||
inputTokens: result.usage?.inputTokens ?? 0,
|
||||
outputTokens: result.usage?.outputTokens ?? 0,
|
||||
},
|
||||
],
|
||||
config: {
|
||||
outputDimensionality: 1536,
|
||||
},
|
||||
})
|
||||
|
||||
const values = response.embeddings?.[0]?.values
|
||||
if (!values) {
|
||||
throw new Error('No embedding values returned from gemini-embedding-2')
|
||||
}
|
||||
|
||||
return {
|
||||
embedding: values,
|
||||
usage,
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof ApplicationFailure) {
|
||||
throw err
|
||||
}
|
||||
throw ApplicationFailure.create({
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
nonRetryable: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -315,7 +224,7 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
|
||||
user: { teamMembers: { some: { teamId: params.teamId } } },
|
||||
},
|
||||
})
|
||||
if (!agent) {
|
||||
if (!agent || process.env.ENABLE_EMBEDDING_AGENT === 'false') {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'embedding feature is disabled or agent not found',
|
||||
nonRetryable: true,
|
||||
@@ -339,12 +248,51 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
|
||||
|
||||
const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '')
|
||||
const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
|
||||
const embeddingProvider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
|
||||
const existingEmbeddings = await prisma.$queryRaw<ExistingAssetEmbeddingInfo[]>`
|
||||
SELECT
|
||||
id,
|
||||
start_time as "startTime",
|
||||
end_time as "endTime",
|
||||
(embedding IS NOT NULL) as "has1536",
|
||||
(embedding_768 IS NOT NULL) as "has768"
|
||||
FROM asset_embeddings
|
||||
WHERE asset_id = ${params.assetId}
|
||||
`
|
||||
|
||||
return {
|
||||
agent,
|
||||
asset,
|
||||
chunkDuration,
|
||||
chunkOverlap,
|
||||
embeddingProvider,
|
||||
existingEmbeddings,
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExistingAssetEmbeddingInfo {
|
||||
id: string
|
||||
startTime: number | null
|
||||
endTime: number | null
|
||||
has1536: boolean
|
||||
has768: boolean
|
||||
}
|
||||
|
||||
export async function deleteAssetEmbeddingsActivity(params: { assetId: string; ids?: string[] }) {
|
||||
if (params.ids && params.ids.length > 0) {
|
||||
await prisma.assetEmbedding.deleteMany({
|
||||
where: {
|
||||
assetId: params.assetId,
|
||||
id: { in: params.ids },
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await prisma.assetEmbedding.deleteMany({
|
||||
where: {
|
||||
assetId: params.assetId,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -358,16 +306,32 @@ export async function saveAssetEmbeddingsActivity(params: {
|
||||
}) {
|
||||
for (const item of params.embeddings) {
|
||||
const embVec = JSON.stringify(item.embedding)
|
||||
const is768 = item.embedding.length === 768
|
||||
|
||||
if (item.startTime !== undefined && item.endTime !== undefined) {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
|
||||
`
|
||||
if (is768) {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
|
||||
`
|
||||
} else {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
|
||||
`
|
||||
}
|
||||
} else {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
|
||||
`
|
||||
if (is768) {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
|
||||
`
|
||||
} else {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
|
||||
`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -384,6 +348,30 @@ export async function generateImageEmbeddingActivity(
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.assetKey,
|
||||
)
|
||||
|
||||
if (provider === 'gemma-embedding-2-openai') {
|
||||
let imgData = data
|
||||
let mimeType = params.mediaType || 'image/jpeg'
|
||||
if (mimeType !== 'image/jpeg' && mimeType !== 'image/png') {
|
||||
imgData = await sharp(data).jpeg().toBuffer()
|
||||
mimeType = 'image/jpeg'
|
||||
}
|
||||
return callGemmaEmbeddingServer([
|
||||
{
|
||||
type: 'image_url',
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url: {
|
||||
url: `data:${mimeType};base64,${imgData.toString('base64')}`,
|
||||
},
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
@@ -401,10 +389,6 @@ export async function generateImageEmbeddingActivity(
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.assetKey,
|
||||
)
|
||||
const embVec = await generateMultimodalEmbedding(ai, data, params.mediaType)
|
||||
|
||||
return {
|
||||
@@ -413,6 +397,38 @@ export async function generateImageEmbeddingActivity(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard aspect ratios for video embedding:
|
||||
* 1:1, 4:3, 3:4, 16:9, 9:16, 21:9 (including ultrawide 64:27), and 9:21.
|
||||
*
|
||||
* For these standard ratios, 15 frames guarantees that total input tokens stay strictly
|
||||
* within the 2048 token signature bucket (<= 1982 tokens). For non-standard or custom
|
||||
* aspect ratios (e.g. 5:3, 1.4:1), 14 frames is used to mathematically guarantee
|
||||
* that sequence length never exceeds 2048 tokens (<= 1990 tokens), preventing OOM errors.
|
||||
*/
|
||||
export function getGemmaEmbeddingTargetFrameCount(width?: number, height?: number): number {
|
||||
if (!width || !height || width <= 0 || height <= 0) {
|
||||
return 14
|
||||
}
|
||||
|
||||
const ratio = width / height
|
||||
const standardRatios = [
|
||||
1 / 1, // 1:1
|
||||
4 / 3, // 4:3
|
||||
3 / 4, // 3:4
|
||||
16 / 9, // 16:9
|
||||
9 / 16, // 9:16
|
||||
21 / 9, // 21:9
|
||||
64 / 27, // 21:9 ultrawide (2560x1080, 3440x1440)
|
||||
9 / 21, // 9:21
|
||||
27 / 64, // 9:21 ultrawide portrait
|
||||
]
|
||||
|
||||
// Allow up to 3% relative tolerance for minor encoder rounding (e.g. 854x480 vs 16:9)
|
||||
const isStandard = standardRatios.some((std) => Math.abs(ratio - std) / std <= 0.03)
|
||||
return isStandard ? 15 : 14
|
||||
}
|
||||
|
||||
export interface GenerateVideoChunkEmbeddingParams {
|
||||
teamId: string
|
||||
chunkKey: string
|
||||
@@ -424,6 +440,83 @@ export async function generateVideoChunkEmbeddingActivity(
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
const isGemma = provider === 'gemma-embedding-2-openai' || params.chunkKey.endsWith('.tar')
|
||||
|
||||
if (isGemma) {
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.chunkKey,
|
||||
)
|
||||
|
||||
let filesMap: Map<string, File>
|
||||
try {
|
||||
const archive = new Bun.Archive(data)
|
||||
filesMap = await archive.files()
|
||||
} catch (err) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `Failed to read chunk archive ${params.chunkKey}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const frameFiles = Array.from(filesMap.keys())
|
||||
.filter((f) => /\.(jpe?g|png|webp)$/i.test(f))
|
||||
.sort()
|
||||
|
||||
if (frameFiles.length === 0) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `No extracted frames found in chunk archive ${params.chunkKey}`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
let targetFrameCount = 14
|
||||
try {
|
||||
const firstFile = filesMap.get(frameFiles[0])
|
||||
if (firstFile) {
|
||||
const firstFileBytes = await firstFile.bytes()
|
||||
if (firstFileBytes.byteLength > 0) {
|
||||
const meta = await sharp(Buffer.from(firstFileBytes)).metadata()
|
||||
targetFrameCount = getGemmaEmbeddingTargetFrameCount(meta.width, meta.height)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
targetFrameCount = 14
|
||||
}
|
||||
|
||||
let selectedFrames = frameFiles
|
||||
if (selectedFrames.length > targetFrameCount) {
|
||||
const step = (selectedFrames.length - 1) / (targetFrameCount - 1)
|
||||
selectedFrames = Array.from(
|
||||
{ length: targetFrameCount },
|
||||
(_, i) => selectedFrames[Math.round(i * step)],
|
||||
)
|
||||
}
|
||||
|
||||
const input: OpenAiEmbeddingItem[] = await Promise.all(
|
||||
selectedFrames.map(async (f) => {
|
||||
const file = filesMap.get(f)!
|
||||
const fileBytes = await file.bytes()
|
||||
if (fileBytes.byteLength === 0) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `Extracted frame ${f} in chunk archive ${params.chunkKey} is empty (0 bytes)`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
return {
|
||||
type: 'image_url',
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url: {
|
||||
url: `data:image/jpeg;base64,${Buffer.from(fileBytes).toString('base64')}`,
|
||||
},
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
return await callGemmaEmbeddingServer(input)
|
||||
}
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
|
||||
@@ -350,5 +350,24 @@ describe('serializeContextToXml', () => {
|
||||
'<context>\n <annotation id="test<id>&"123"" />\n</context>',
|
||||
)
|
||||
})
|
||||
|
||||
it('serializes position type="degree" with integer degree attribute without seconds', () => {
|
||||
const context: ShumaiMessageContext = {
|
||||
currentAsset: {
|
||||
id: 'ast_3d',
|
||||
name: 'robot.glb',
|
||||
type: 'file',
|
||||
mediaType: '3d',
|
||||
path: 'robot.glb',
|
||||
},
|
||||
position: {
|
||||
type: 'degree',
|
||||
degree: 45,
|
||||
},
|
||||
}
|
||||
expect(serializeContextToXml(context)).toBe(
|
||||
'<context>\n <current_asset id="ast_3d" name="robot.glb" type="file" media_type="3d" path="robot.glb" />\n <position type="degree" degree="45" />\n</context>',
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -610,11 +610,9 @@ import * as aiActivities from './activities/ai'
|
||||
import { agentAutofillMedia } from './workflows/agent-autofill'
|
||||
import { agentChat } from './workflows/agent-chat'
|
||||
import { agentEmbeddingMedia } from './workflows/agent-embedding'
|
||||
import { queryEmbeddingForSearch } from './workflows/query-embedding-for-search'
|
||||
|
||||
export function initAgentWorkflows() {
|
||||
registerWorkflow(WorkflowTaskType.ai_embedding, agentEmbeddingMedia)
|
||||
registerWorkflow(WorkflowTaskType.query_embedding_for_search, queryEmbeddingForSearch)
|
||||
registerWorkflow(WorkflowTaskType.ai_metadata_autofill, agentAutofillMedia)
|
||||
registerWorkflow(WorkflowTaskType.chat, agentChat)
|
||||
|
||||
@@ -625,21 +623,21 @@ export function initAgentWorkflows() {
|
||||
export * from './activities/agent'
|
||||
export {
|
||||
extractAiMetadataActivity,
|
||||
generateEmbeddingActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateTextEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
deleteAssetEmbeddingsActivity,
|
||||
type ExtractAiMetadataParams,
|
||||
type GeneratedEmbedding,
|
||||
type GenerateEmbeddingParams,
|
||||
type GenerateTextEmbeddingParams,
|
||||
type ExistingAssetEmbeddingInfo,
|
||||
} from './activities/ai'
|
||||
export * from './database-session-storage'
|
||||
export * from './context/serialize-context'
|
||||
export * from './workflows/agent-autofill'
|
||||
export * from './workflows/agent-chat'
|
||||
export * from './workflows/agent-embedding'
|
||||
export * from './workflows/query-embedding-for-search'
|
||||
export * from './tools/generate-image'
|
||||
export * from './tools/generate-video'
|
||||
export * from './tools/rename-asset'
|
||||
|
||||
@@ -4,6 +4,7 @@ import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { resolveAnnotationsById } from './annotation-resolver'
|
||||
import { DatabaseSessionStorage } from '../database-session-storage'
|
||||
import type { SessionTreeEntry } from '@earendil-works/pi-agent-core'
|
||||
import { degreeToSecond } from '@shumai/core/src/utils/model3d'
|
||||
|
||||
describe('resolveAnnotationsById', () => {
|
||||
setupTestDbHooks()
|
||||
@@ -210,6 +211,89 @@ describe('resolveAnnotationsById', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('resolves annotations and degree position timestamp from 1-on-1 chat AgentSessionEntry for 3D assets', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User 3D', email: 'user3d@test.com', password: 'pw' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team 3D' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project 3D', teamId: team.id },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: 'file',
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
const agentUser = await prisma.user.create({
|
||||
data: { name: 'Agent 3D', email: 'agent3d@example.com', type: 'agent' },
|
||||
})
|
||||
const agent = await prisma.agent.create({
|
||||
data: {
|
||||
id: agentUser.id,
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'test-provider', model: 'test-model' },
|
||||
},
|
||||
})
|
||||
|
||||
const storage = await DatabaseSessionStorage.create({
|
||||
agentId: agent.id,
|
||||
userId: user.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
|
||||
storage.currentMessageContext = {
|
||||
currentAsset: {
|
||||
id: asset.id,
|
||||
name: asset.name,
|
||||
type: 'file',
|
||||
path: asset.name,
|
||||
},
|
||||
position: { type: 'degree', degree: 45 },
|
||||
annotation: true,
|
||||
annotations: [
|
||||
{
|
||||
type: 'rectangle',
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 30,
|
||||
height: 40,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const userEntry: SessionTreeEntry = {
|
||||
type: 'message',
|
||||
id: '01JTEST3DENTRY123456789',
|
||||
parentId: null,
|
||||
timestamp: new Date().toISOString(),
|
||||
message: {
|
||||
role: 'user',
|
||||
content: 'Check markup at 45 degrees',
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
}
|
||||
|
||||
await storage.appendEntry(userEntry)
|
||||
|
||||
const result = await resolveAnnotationsById(asset.id, '01JTEST3DENTRY123456789')
|
||||
expect(result.timestamp).toBeCloseTo(degreeToSecond(45))
|
||||
expect(result.annotations).toEqual([
|
||||
{
|
||||
type: 'rectangle',
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 30,
|
||||
height: 40,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('throws an error when AgentSessionEntry belongs to a different asset', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'User', email: 'user3@test.com', password: 'pw' },
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import type { ShumaiMessageContext } from '@shumai/dtos'
|
||||
import { degreeToSecond } from '@shumai/core/src/utils/model3d'
|
||||
|
||||
export async function resolveAnnotationsById(
|
||||
assetId: string,
|
||||
@@ -44,7 +45,14 @@ export async function resolveAnnotationsById(
|
||||
const list = details?.annotations
|
||||
const annotations =
|
||||
Array.isArray(list) && list.length > 0 ? (list as unknown as PrismaJson.AnnotationList) : null
|
||||
const timestamp = details?.position?.type === 'time' ? details.position.seconds : null
|
||||
let timestamp: number | null = null
|
||||
if (details?.position?.type === 'time') {
|
||||
timestamp = details.position.seconds
|
||||
} else if (details?.position?.type === 'degree') {
|
||||
timestamp = degreeToSecond(details.position.degree)
|
||||
} else if (details?.position?.type === 'page') {
|
||||
timestamp = details.position.page
|
||||
}
|
||||
return { annotations, timestamp }
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
import * as annotationResolver from './annotation-resolver'
|
||||
import { degreeToSecond } from '@shumai/core/src/utils/model3d'
|
||||
|
||||
vi.mock('@shumai/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@shumai/db')>()
|
||||
@@ -211,6 +212,476 @@ describe('readAssetTool', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('3D Assets', () => {
|
||||
it('should read 3D asset 4-views image and return image with quadrant layout description', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
storageKey: { key: 'raw/robot.glb' },
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [{ key: 'proxy/views.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-views-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-3d', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith({
|
||||
user: { id: 'user-1' },
|
||||
permission: 'Read',
|
||||
type: 'asset',
|
||||
id: 'model-1',
|
||||
})
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/views.webp')
|
||||
|
||||
const textItem = result.content[0] as { type: 'text'; text: string }
|
||||
const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
|
||||
|
||||
expect(textItem.text).toContain('Orthogonal 4-view composite image')
|
||||
expect(textItem.text).toContain('Top-Left quadrant: Front view (0°)')
|
||||
expect(textItem.text).toContain('Top-Right quadrant: Right side view (90°)')
|
||||
expect(textItem.text).toContain('Bottom-Left quadrant: Back view (180°)')
|
||||
expect(textItem.text).toContain('Bottom-Right quadrant: Left side view (270°)')
|
||||
expect(imageItem.type).toBe('image')
|
||||
expect(imageItem.data).toBe(Buffer.from('fake-views-bytes').toString('base64'))
|
||||
expect(result.details.key).toBe('proxy/views.webp')
|
||||
})
|
||||
|
||||
it('should support s3KeyOnly mode for 3D assets', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [{ key: 'proxy/views.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-3d-s3', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
s3KeyOnly: true,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(result.content).toHaveLength(1)
|
||||
const textItem = result.content[0] as { type: 'text'; text: string }
|
||||
expect(textItem.text).toContain('proxy/views.webp')
|
||||
expect(textItem.text).toContain('Front view (0°)')
|
||||
expect(result.details.key).toBe('proxy/views.webp')
|
||||
})
|
||||
|
||||
it('should throw error if videoConfig is passed for 3D assets', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [{ key: 'proxy/views.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-3d-err', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: { start: 0, end: 4, count: 4 },
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig",
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if 3D asset imageTranscodes is missing when overview is requested', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-3d-err2', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('4-views image (views.webp) is missing')
|
||||
})
|
||||
|
||||
it('should extract specific turntable frame when config3d.degree is provided', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
projectId: 'proj-1',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
|
||||
imageTranscodes: [{ key: 'proxy/views.webp' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-shot-3d',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-shot-3d',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
screenshots: [{ key: 'proxy/shot-45.webp', timestamp: 0.5833 }],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-shot-45-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-3d-degree', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
config3d: { degree: 45 },
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
assetId: 'model-1',
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
screenshot: {
|
||||
start: expect.any(Number),
|
||||
end: expect.any(Number),
|
||||
count: 1,
|
||||
commentTimestamp: expect.any(Number),
|
||||
annotations: undefined,
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
const textItem = result.content[0] as { type: 'text'; text: string }
|
||||
const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
|
||||
|
||||
expect(textItem.text).toContain('view at 45°:')
|
||||
expect(imageItem.data).toBe(Buffer.from('fake-shot-45-bytes').toString('base64'))
|
||||
expect(result.details.key).toBe('proxy/shot-45.webp')
|
||||
})
|
||||
|
||||
it('should throw error when config3d.degree is not a multiple of 15 or out of range', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
|
||||
await expect(
|
||||
tool.execute('call-3d-invalid-deg', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
config3d: { degree: 40 },
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('Invalid turntable degree: 40')
|
||||
|
||||
await expect(
|
||||
tool.execute('call-3d-out-of-range', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
config3d: { degree: 360 },
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('Invalid turntable degree: 360')
|
||||
})
|
||||
|
||||
it('should extract frame with overlaid markups when annotationId is provided on 3D asset', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
projectId: 'proj-1',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
|
||||
annotations: [
|
||||
{
|
||||
type: 'freehand',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[10, 10],
|
||||
],
|
||||
},
|
||||
],
|
||||
timestamp: 0.5, // 0.5s = 45°
|
||||
})
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-shot-ann',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-shot-ann',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
screenshots: [{ key: 'proxy/shot-annotated.webp', timestamp: 0.5 }],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-shot-ann-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-3d-ann', {
|
||||
assetId: 'model-1',
|
||||
annotationId: 'comm-1',
|
||||
config3d: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
assetId: 'model-1',
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
screenshot: {
|
||||
start: 0.5,
|
||||
end: 0.5,
|
||||
count: 1,
|
||||
commentTimestamp: 0.5,
|
||||
annotations: [
|
||||
{
|
||||
type: 'freehand',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[10, 10],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
const textItem = result.content[0] as { type: 'text'; text: string }
|
||||
expect(textItem.text).toContain('view at 45° with visual markups overlaid:')
|
||||
expect(result.details.key).toBe('proxy/shot-annotated.webp')
|
||||
})
|
||||
|
||||
it('should extract turntable frame at resolved degree angle for 3D session annotations', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
projectId: 'proj-1',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const expectedTime = degreeToSecond(90)
|
||||
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#00ff00',
|
||||
points: [
|
||||
[10, 20],
|
||||
[40, 60],
|
||||
],
|
||||
},
|
||||
],
|
||||
timestamp: expectedTime,
|
||||
})
|
||||
|
||||
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
|
||||
id: 'task-shot-3d-session',
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(workflowService.executeWait).mockResolvedValue({
|
||||
id: 'task-shot-3d-session',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
output: {
|
||||
screenshots: [{ key: 'proxy/shot-annotated-90.webp', timestamp: expectedTime }],
|
||||
},
|
||||
} as unknown as WorkflowTask)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-shot-90-bytes'),
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
const result = await tool.execute('call-3d-session-ann', {
|
||||
assetId: 'model-1',
|
||||
annotationId: 'session-entry-1',
|
||||
config3d: null,
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
})
|
||||
|
||||
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
assetId: 'model-1',
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
screenshot: {
|
||||
start: expectedTime,
|
||||
end: expectedTime,
|
||||
count: 1,
|
||||
commentTimestamp: expectedTime,
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#00ff00',
|
||||
points: [
|
||||
[10, 20],
|
||||
[40, 60],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
const textItem = result.content[0] as { type: 'text'; text: string }
|
||||
expect(textItem.text).toContain('view at 90° with visual markups overlaid:')
|
||||
expect(result.details.key).toBe('proxy/shot-annotated-90.webp')
|
||||
})
|
||||
|
||||
it('should throw error if config3d.degree conflicts with annotation angle', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
projectId: 'proj-1',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
|
||||
annotations: [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ff0000',
|
||||
points: [
|
||||
[0, 0],
|
||||
[10, 10],
|
||||
],
|
||||
},
|
||||
],
|
||||
timestamp: 0.5, // 45°
|
||||
})
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-3d-conflict', {
|
||||
assetId: 'model-1',
|
||||
annotationId: 'comm-1',
|
||||
config3d: { degree: 90 }, // Conflicts with 45°
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
'Annotation "comm-1" was drawn at 45°, but degree 90° was specified. Do not pass a conflicting degree with annotationId.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw error if 3D asset turntable video is missing when angle inspection is requested', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'model-1',
|
||||
name: 'robot.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [],
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadAssetTool('user-1')
|
||||
await expect(
|
||||
tool.execute('call-3d-missing-turntable', {
|
||||
assetId: 'model-1',
|
||||
annotationId: null,
|
||||
config3d: { degree: 45 },
|
||||
imageConfig: null,
|
||||
videoConfig: null,
|
||||
docConfig: null,
|
||||
}),
|
||||
).rejects.toThrow('turntable video (turntable.mp4) is missing')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Video Assets', () => {
|
||||
it('should throw error if videoConfig is missing for video assets', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
|
||||
@@ -7,6 +7,12 @@ import { workflowService } from '@shumai/workflow-core'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
import { resolveAnnotationsById } from './annotation-resolver'
|
||||
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
|
||||
import { is3dModel } from '@shumai/core/src/utils/mime'
|
||||
import {
|
||||
isValidTurntableDegree,
|
||||
secondToDegree,
|
||||
degreeToSecond,
|
||||
} from '@shumai/core/src/utils/model3d'
|
||||
|
||||
export const readAssetSchema = Type.Object(
|
||||
{
|
||||
@@ -37,6 +43,31 @@ export const readAssetSchema = Type.Object(
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
config3d: Type.Optional(
|
||||
Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
degree: Type.Optional(
|
||||
Type.Union([
|
||||
Type.Integer({
|
||||
minimum: 0,
|
||||
maximum: 345,
|
||||
description:
|
||||
'Turntable rotation degree (integer: 0, 15, 30, ..., 345 in 15° increments) to inspect a specific angle of a 3D model. Leave null/omitted to view the 4-view composite overview.',
|
||||
}),
|
||||
Type.Null(),
|
||||
]),
|
||||
),
|
||||
},
|
||||
{
|
||||
additionalProperties: false,
|
||||
description:
|
||||
'Configuration for 3D model assets. Pass null if inspecting another asset type. Optionally pass degree to inspect a specific turntable angle, or leave null/omitted to view the 4-view composite overview.',
|
||||
},
|
||||
),
|
||||
Type.Null(),
|
||||
]),
|
||||
),
|
||||
videoConfig: Type.Union([
|
||||
Type.Object(
|
||||
{
|
||||
@@ -167,12 +198,12 @@ export function createReadAssetTool(
|
||||
label: 'Read Asset',
|
||||
description:
|
||||
'Inspects and reads the content of an asset in the workspace. ' +
|
||||
'Supports visual analysis for images, video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
|
||||
'Pass assetId, and provide the specific config (imageConfig, videoConfig, or docConfig) while setting unused configs to null. ' +
|
||||
'Supports visual analysis for images, 3D models (overview or specific degree angle), video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
|
||||
'Pass assetId, and provide the specific config (imageConfig, config3d, videoConfig, or docConfig) while setting unused configs to null. ' +
|
||||
'Optionally provide annotationId to view visual markups drawn on the asset.',
|
||||
parameters: readAssetSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const { assetId, annotationId, s3KeyOnly, videoConfig, docConfig } = params
|
||||
const { assetId, annotationId, s3KeyOnly, config3d, videoConfig, docConfig } = params
|
||||
|
||||
if (agentType === 'autofill' && !userId) {
|
||||
if (!targetAssetId) {
|
||||
@@ -243,6 +274,209 @@ export function createReadAssetTool(
|
||||
const filename = asset.name?.toLowerCase() || ''
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Branch 0: 3D Assets (Orthogonal 4-View Composite Image or Degree Angle)
|
||||
// ----------------------------------------------------------------------
|
||||
const is3d = proxyType === '3d' || is3dModel(mediaType, filename)
|
||||
if (is3d) {
|
||||
if (videoConfig) {
|
||||
throw new Error(
|
||||
`Asset ${asset.id} is a 3D model. To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig.`,
|
||||
)
|
||||
}
|
||||
|
||||
if (config3d?.degree !== undefined && config3d?.degree !== null) {
|
||||
if (!isValidTurntableDegree(config3d.degree)) {
|
||||
throw new Error(
|
||||
`Invalid turntable degree: ${config3d.degree}. Degree must be an integer multiple of 15 between 0 and 345 (e.g. 0, 15, 30, 45, ..., 345).`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let annotations: PrismaJson.AnnotationList | null = null
|
||||
let commentTimestamp: number | null = null
|
||||
if (annotationId) {
|
||||
const resolved = await resolveAnnotationsById(asset.id, annotationId)
|
||||
annotations = resolved.annotations
|
||||
commentTimestamp = resolved.timestamp
|
||||
if (commentTimestamp !== null && commentTimestamp !== undefined) {
|
||||
const commentDegree = secondToDegree(commentTimestamp)
|
||||
if (
|
||||
config3d?.degree !== undefined &&
|
||||
config3d?.degree !== null &&
|
||||
config3d.degree !== commentDegree
|
||||
) {
|
||||
throw new Error(
|
||||
`Annotation "${annotationId}" was drawn at ${commentDegree}°, but degree ${config3d.degree}° was specified. Do not pass a conflicting degree with annotationId.`,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((config3d?.degree !== undefined && config3d?.degree !== null) || annotationId) {
|
||||
if (!mediaInfo?.videoTranscodes || mediaInfo.videoTranscodes.length === 0) {
|
||||
throw new Error(
|
||||
`Asset ${asset.id} is a 3D model, but its turntable video (turntable.mp4) is missing.`,
|
||||
)
|
||||
}
|
||||
|
||||
let targetDegree: number
|
||||
let targetSecond: number
|
||||
|
||||
if (config3d?.degree !== undefined && config3d?.degree !== null) {
|
||||
targetDegree = config3d.degree
|
||||
targetSecond = degreeToSecond(targetDegree)
|
||||
} else {
|
||||
targetSecond =
|
||||
commentTimestamp !== null && commentTimestamp !== undefined ? commentTimestamp : 0
|
||||
targetDegree = secondToDegree(targetSecond)
|
||||
}
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
screenshot: {
|
||||
start: targetSecond,
|
||||
end: targetSecond,
|
||||
count: 1,
|
||||
commentTimestamp: targetSecond,
|
||||
annotations: annotations ?? undefined,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode screenshot output
|
||||
const output = completedTask.output as any
|
||||
const screenshots = output?.screenshots as
|
||||
| Array<{ key: string; timestamp: number }>
|
||||
| undefined
|
||||
|
||||
if (!screenshots || screenshots.length === 0) {
|
||||
throw new Error(
|
||||
`Failed to extract 3D turntable frame for asset ${asset.id} at ${targetDegree}°.`,
|
||||
)
|
||||
}
|
||||
|
||||
const shotKey = screenshots[0].key
|
||||
const downloadUrl = await s3Service.presign(
|
||||
bucket,
|
||||
shotKey,
|
||||
'GET',
|
||||
true,
|
||||
asset.name || undefined,
|
||||
)
|
||||
|
||||
const content: Array<TextContent | ImageContent> = []
|
||||
const hasMarkup = Boolean(annotations && annotations.length > 0)
|
||||
const textDescription = hasMarkup
|
||||
? `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}° with visual markups overlaid:`
|
||||
: `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}°:`
|
||||
|
||||
if (s3KeyOnly) {
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: textDescription,
|
||||
})
|
||||
} else {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, shotKey)
|
||||
const actualMimeType =
|
||||
contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp'
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: textDescription,
|
||||
})
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: actualMimeType,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
key: shotKey,
|
||||
downloadUrl,
|
||||
sourceKeys: [shotKey],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (!mediaInfo?.imageTranscodes || mediaInfo.imageTranscodes.length === 0) {
|
||||
throw new Error(
|
||||
`Asset ${asset.id} is a 3D model, but its 4-views image (views.webp) is missing.`,
|
||||
)
|
||||
}
|
||||
|
||||
const mediaKey = mediaInfo.imageTranscodes[0].key
|
||||
if (!mediaKey) {
|
||||
throw new Error(`Asset ${asset.id} is a 3D model, but its 4-views image key is missing.`)
|
||||
}
|
||||
|
||||
const keyToUse = mediaKey
|
||||
const mimeType = 'image/webp'
|
||||
|
||||
const downloadUrl = await s3Service.presign(
|
||||
bucket,
|
||||
keyToUse,
|
||||
'GET',
|
||||
true,
|
||||
asset.name || undefined,
|
||||
)
|
||||
|
||||
const content: Array<TextContent | ImageContent> = []
|
||||
const textDescription = [
|
||||
`3D asset "${asset.name}" (ID: ${asset.id}, MIME: ${mimeType}, S3 Key: "${keyToUse}"):`,
|
||||
'Orthogonal 4-view composite image (2x2 grid across 360° rotation):',
|
||||
'- Top-Left quadrant: Front view (0°)',
|
||||
'- Top-Right quadrant: Right side view (90°)',
|
||||
'- Bottom-Left quadrant: Back view (180°)',
|
||||
'- Bottom-Right quadrant: Left side view (270°)',
|
||||
].join('\n')
|
||||
|
||||
if (s3KeyOnly) {
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: textDescription,
|
||||
})
|
||||
} else {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
|
||||
const actualMimeType =
|
||||
contentType && contentType !== 'application/octet-stream' ? contentType : mimeType
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: textDescription,
|
||||
})
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: actualMimeType,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
assetId: asset.id,
|
||||
name: asset.name,
|
||||
mediaType: asset.mediaType,
|
||||
key: keyToUse,
|
||||
downloadUrl,
|
||||
sourceKeys: [keyToUse],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Branch 1: Image Assets
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
@@ -129,6 +129,42 @@ describe('createReadThreadTool', () => {
|
||||
expect(textContent).toContain('- [Ai Agent] (id: reply-1, has_markup: false): Agent answer')
|
||||
})
|
||||
|
||||
it('should format 3d asset comment timestamps as degree: X°', async () => {
|
||||
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
|
||||
id: 'root-3d',
|
||||
message: 'Notice this angle',
|
||||
creator: { name: 'Alice' },
|
||||
second: 0.5,
|
||||
annotation: [{ type: 'box' }],
|
||||
asset: {
|
||||
media: { proxyType: '3d' },
|
||||
mediaType: 'model/gltf-binary',
|
||||
name: 'robot.glb',
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
vi.mocked(prisma.assetComment.findMany).mockResolvedValue([
|
||||
{
|
||||
id: 'reply-3d-1',
|
||||
message: 'Looking at back side',
|
||||
creator: { name: 'Bob' },
|
||||
second: 2.0,
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
] as any)
|
||||
|
||||
const tool = createReadThreadTool()
|
||||
const result = await tool.execute('call-3d', { threadId: 'root-3d' })
|
||||
const textContent = (result.content[0] as { text: string }).text
|
||||
|
||||
expect(textContent).toContain(
|
||||
'Thread Root [Alice] (id: root-3d, degree: 45°, has_markup: true): Notice this angle',
|
||||
)
|
||||
expect(textContent).toContain(
|
||||
'- [Bob] (id: reply-3d-1, degree: 180°, has_markup: false): Looking at back side',
|
||||
)
|
||||
})
|
||||
|
||||
it('should describe threadId parameter as referencing <thread id="..." /> in <context>', () => {
|
||||
const tool = createReadThreadTool()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { Type } from 'typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import { secondToDegree } from '@shumai/core/src/utils/model3d'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
|
||||
const readThreadSchema = Type.Object({
|
||||
threadId: Type.String({
|
||||
@@ -9,14 +11,21 @@ const readThreadSchema = Type.Object({
|
||||
}),
|
||||
})
|
||||
|
||||
function formatCommentMeta(c: {
|
||||
id: string
|
||||
second?: number | null
|
||||
annotation?: unknown
|
||||
}): string {
|
||||
function formatCommentMeta(
|
||||
c: {
|
||||
id: string
|
||||
second?: number | null
|
||||
annotation?: unknown
|
||||
},
|
||||
proxyType?: string,
|
||||
): string {
|
||||
const parts = [`id: ${c.id}`]
|
||||
if (c.second !== null && c.second !== undefined) {
|
||||
parts.push(`time: ${c.second}s`)
|
||||
if (proxyType === '3d') {
|
||||
parts.push(`degree: ${secondToDegree(c.second)}°`)
|
||||
} else {
|
||||
parts.push(`time: ${c.second}s`)
|
||||
}
|
||||
}
|
||||
const hasMarkup = Boolean(c.annotation && Array.isArray(c.annotation) && c.annotation.length > 0)
|
||||
parts.push(`has_markup: ${hasMarkup}`)
|
||||
@@ -34,7 +43,10 @@ export const createReadThreadTool = (): AgentTool<
|
||||
execute: async (_toolCallId, params) => {
|
||||
const rootComment = await prisma.assetComment.findUnique({
|
||||
where: { id: params.threadId },
|
||||
include: { creator: true },
|
||||
include: {
|
||||
creator: true,
|
||||
asset: { select: { media: true, mediaType: true, name: true } },
|
||||
},
|
||||
})
|
||||
|
||||
if (!rootComment) {
|
||||
@@ -61,9 +73,14 @@ export const createReadThreadTool = (): AgentTool<
|
||||
include: { creator: true },
|
||||
})
|
||||
|
||||
const proxyType =
|
||||
(rootComment.asset?.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(rootComment.asset?.mediaType, rootComment.asset?.name) ||
|
||||
undefined
|
||||
|
||||
const isRootAgent = rootComment.creator?.type === 'agent' || rootComment.sessionId !== null
|
||||
const rootAuthor = rootComment.creator?.name || (isRootAgent ? 'Ai Agent' : 'User')
|
||||
const rootMeta = formatCommentMeta(rootComment)
|
||||
const rootMeta = formatCommentMeta(rootComment, proxyType)
|
||||
let output = `Thread Root [${rootAuthor}] ${rootMeta}: ${rootComment.message || ''}\n\nReplies (${replies.length}):`
|
||||
|
||||
if (replies.length === 0) {
|
||||
@@ -72,7 +89,7 @@ export const createReadThreadTool = (): AgentTool<
|
||||
for (const reply of replies) {
|
||||
const isAgent = reply.creator?.type === 'agent' || reply.sessionId !== null
|
||||
const author = reply.creator?.name || (isAgent ? 'Ai Agent' : 'User')
|
||||
const replyMeta = formatCommentMeta(reply)
|
||||
const replyMeta = formatCommentMeta(reply, proxyType)
|
||||
output += `\n- [${author}] ${replyMeta}: ${reply.message || ''}`
|
||||
}
|
||||
}
|
||||
|
||||
@@ -806,4 +806,89 @@ describe('Agent Chat Workflow', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should format mediaPosition as degree for 3D assets in 1-on-1 chat', async () => {
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'a-3d',
|
||||
project: { teamId: 't1' },
|
||||
type: 'file',
|
||||
name: 'model.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: { proxyType: '3d' },
|
||||
parentId: null,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: 'a-3d',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
agent: {
|
||||
agentId: 'b1',
|
||||
prompt: 'Look at this angle',
|
||||
sessionId: 'session-3d-1',
|
||||
second: 0.5, // 0.5s turntable time = 45°
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await agentChat(task)
|
||||
|
||||
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
messageContext: expect.objectContaining({
|
||||
position: { type: 'degree', degree: 45 },
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should format mediaPosition as degree for 3D assets in comment mode', async () => {
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'a-3d',
|
||||
project: { teamId: 't1' },
|
||||
type: 'file',
|
||||
name: 'model.glb',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: { proxyType: '3d' },
|
||||
parentId: null,
|
||||
})
|
||||
|
||||
mockActivities.getCommentActivity.mockResolvedValue({
|
||||
id: 'comment-3d-1',
|
||||
creatorId: 'user-c1',
|
||||
message: 'Look at the back',
|
||||
replyToId: null,
|
||||
second: 2.0, // 2.0s = 180°
|
||||
attachments: [],
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'chat',
|
||||
status: 'pending',
|
||||
assetId: 'a-3d',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
agent: {
|
||||
agentId: 'b1',
|
||||
userCommentId: 'comment-3d-1',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await agentChat(task)
|
||||
|
||||
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
messageContext: expect.objectContaining({
|
||||
position: { type: 'degree', degree: 180 },
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -8,6 +8,7 @@ import type {
|
||||
ShumaiMediaPosition,
|
||||
} from '@shumai/dtos'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
import { secondToDegree } from '@shumai/core/src/utils/model3d'
|
||||
|
||||
export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
const {
|
||||
@@ -273,7 +274,9 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
// Build position
|
||||
let mediaPosition: ShumaiMediaPosition | undefined
|
||||
if (commentTimestamp !== undefined && commentTimestamp !== null) {
|
||||
if (proxyType === 'pdf') {
|
||||
if (proxyType === '3d') {
|
||||
mediaPosition = { type: 'degree', degree: secondToDegree(commentTimestamp) }
|
||||
} else if (proxyType === 'pdf') {
|
||||
mediaPosition = { type: 'page', page: Math.round(commentTimestamp) }
|
||||
} else {
|
||||
mediaPosition = { type: 'time', seconds: commentTimestamp }
|
||||
|
||||
@@ -38,6 +38,9 @@ describe('Agent Embedding Workflow', () => {
|
||||
saveAssetEmbeddingsActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'saveAssetEmbeddingsActivity',
|
||||
}),
|
||||
deleteAssetEmbeddingsActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'deleteAssetEmbeddingsActivity',
|
||||
}),
|
||||
updateTaskUsageActivity: Object.assign(vi.fn(), { _activityName: 'updateTaskUsageActivity' }),
|
||||
createCommentActivity: Object.assign(vi.fn(), { _activityName: 'createCommentActivity' }),
|
||||
updateCommentActivity: Object.assign(vi.fn(), { _activityName: 'updateCommentActivity' }),
|
||||
@@ -211,6 +214,68 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow successfully for 3D asset using 4-views image', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'model-1',
|
||||
mediaType: 'model/gltf-binary',
|
||||
storageKey: { key: 'raw/model.glb' },
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [{ key: 'files/views.webp', format: 'webp' }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'model-1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({
|
||||
teamId: 't1',
|
||||
assetKey: 'files/views.webp',
|
||||
mediaType: 'image/webp',
|
||||
})
|
||||
})
|
||||
|
||||
it('should reject 3D asset for embeddings if imageTranscodes is missing', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'model-1',
|
||||
mediaType: 'model/gltf-binary',
|
||||
storageKey: { key: 'raw/model.glb' },
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'model-1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(agentEmbeddingMedia(task)).rejects.toThrow(
|
||||
'unsupported media type for embeddings: model/gltf-binary',
|
||||
)
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow successfully for video in chunks', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
@@ -252,18 +317,21 @@ describe('Agent Embedding Workflow', () => {
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
format: 'video',
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
|
||||
assetId: 'a1',
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
startTime: 55,
|
||||
endTime: 115,
|
||||
format: 'video',
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
|
||||
assetId: 'a1',
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
startTime: 110,
|
||||
endTime: 150,
|
||||
format: 'video',
|
||||
})
|
||||
|
||||
// Verify video chunk embedding generation (3 chunks) on agent queue
|
||||
@@ -295,14 +363,19 @@ describe('Agent Embedding Workflow', () => {
|
||||
model: 'gemini-embedding-2',
|
||||
})
|
||||
|
||||
// Verify embeddings saved
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
// Verify embeddings saved per chunk immediately
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(3)
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(1, {
|
||||
assetId: 'a1',
|
||||
embeddings: [
|
||||
{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 },
|
||||
{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 },
|
||||
{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 },
|
||||
],
|
||||
embeddings: [{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 }],
|
||||
})
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(2, {
|
||||
assetId: 'a1',
|
||||
embeddings: [{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 }],
|
||||
})
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(3, {
|
||||
assetId: 'a1',
|
||||
embeddings: [{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 }],
|
||||
})
|
||||
})
|
||||
|
||||
@@ -388,6 +461,49 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should save completed chunks to DB immediately before subsequent chunk failure', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 150.0,
|
||||
videoTranscodes: [{ key: 'test-transcoded.mp4' }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
// Chunk 1 succeeds, Chunk 2 fails
|
||||
mockActivities.generateVideoChunkEmbeddingActivity
|
||||
.mockResolvedValueOnce({
|
||||
embedding: [0.1, 0.2],
|
||||
usage: { inputTokens: 3, outputTokens: 3, model: 'gemini-embedding-2' },
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('Rate limit exceeded at chunk 2'))
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(agentEmbeddingMedia(task)).rejects.toThrow('Rate limit exceeded at chunk 2')
|
||||
|
||||
// Chunk 1 MUST have been saved to the database before chunk 2 failed
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(1)
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
embeddings: [{ embedding: [0.1, 0.2], startTime: 0, endTime: 60 }],
|
||||
})
|
||||
})
|
||||
|
||||
it('should not create redundant chunks when video exactly matches chunk duration', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
@@ -423,4 +539,137 @@ describe('Agent Embedding Workflow', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow for video with gemma-embedding-2-openai requesting frames format', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 30.0,
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
embeddingProvider: 'gemma-embedding-2-openai',
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
startTime: 0,
|
||||
endTime: 30,
|
||||
format: 'frames',
|
||||
}),
|
||||
)
|
||||
expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
model: 'gemma-embedding-2-openai',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should skip already completed chunks when resuming a video embedding task', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 150.0,
|
||||
videoTranscodes: [{ key: 'test-transcoded.mp4' }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
existingEmbeddings: [
|
||||
{
|
||||
id: 'emb-1',
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
has1536: true,
|
||||
has768: false,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Chunk 1 (0-60) should be skipped; only chunk 2 (55-115) and chunk 3 (110-150) should be transcoded and embedded
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(2)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ startTime: 55, endTime: 115 }),
|
||||
)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ startTime: 110, endTime: 150 }),
|
||||
)
|
||||
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('should purge incompatible embeddings before processing', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'test.png' },
|
||||
},
|
||||
embeddingProvider: 'gemma-embedding-2-openai',
|
||||
existingEmbeddings: [
|
||||
{
|
||||
id: 'old-gemini-emb',
|
||||
startTime: null,
|
||||
endTime: null,
|
||||
has1536: true,
|
||||
has768: false,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Should purge incompatible old-gemini-emb
|
||||
expect(mockActivities.deleteAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
ids: ['old-gemini-emb'],
|
||||
})
|
||||
|
||||
// Should generate and save new embedding
|
||||
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalled()
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -13,6 +13,14 @@ export interface GeneratedEmbedding {
|
||||
endTime?: number
|
||||
}
|
||||
|
||||
export interface ExistingAssetEmbeddingInfo {
|
||||
id: string
|
||||
startTime: number | null
|
||||
endTime: number | null
|
||||
has1536: boolean
|
||||
has768: boolean
|
||||
}
|
||||
|
||||
export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
const {
|
||||
updateTaskStatusActivity,
|
||||
@@ -20,6 +28,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
deleteAssetEmbeddingsActivity,
|
||||
updateTaskUsageActivity,
|
||||
createCommentActivity,
|
||||
updateCommentActivity,
|
||||
@@ -62,15 +71,45 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
|
||||
// 1. Fetch Agent Context
|
||||
const context = await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, {
|
||||
const context = (await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, {
|
||||
teamId: task.teamId,
|
||||
assetId: task.assetId,
|
||||
})
|
||||
})) as unknown as {
|
||||
asset: {
|
||||
id: string
|
||||
mediaType: string | null
|
||||
storageKey?: { key: string } | null
|
||||
media?: unknown
|
||||
}
|
||||
chunkDuration: number
|
||||
chunkOverlap: number
|
||||
embeddingProvider: string
|
||||
existingEmbeddings?: ExistingAssetEmbeddingInfo[]
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const { asset, chunkDuration, chunkOverlap } = context as any
|
||||
const { asset, chunkDuration, chunkOverlap, embeddingProvider, existingEmbeddings } = context
|
||||
const isGemmaProvider = embeddingProvider === 'gemma-embedding-2-openai'
|
||||
|
||||
const isImage = asset.mediaType?.startsWith('image/')
|
||||
const existing = existingEmbeddings || []
|
||||
|
||||
// Purge incompatible embeddings (e.g. 1536d when Gemma is used, or 768d when Gemini is used)
|
||||
const incompatibleIds = existing
|
||||
.filter((e) => (isGemmaProvider ? e.has1536 || !e.has768 : e.has768 || !e.has1536))
|
||||
.map((e) => e.id)
|
||||
|
||||
if (incompatibleIds.length > 0) {
|
||||
await executeActivity(agentWorkerQueue, deleteAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
ids: incompatibleIds,
|
||||
})
|
||||
}
|
||||
|
||||
const validExistingChunks = existing.filter((e) => !incompatibleIds.includes(e.id))
|
||||
|
||||
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
|
||||
const is3dWithViews =
|
||||
mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0
|
||||
const isImage = asset.mediaType?.startsWith('image/') || is3dWithViews
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
@@ -110,23 +149,30 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
|
||||
}
|
||||
|
||||
const results: GeneratedEmbedding[] = []
|
||||
const usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
model: isGemmaProvider ? 'gemma-embedding-2-openai' : 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
if (isImage) {
|
||||
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
|
||||
teamId: task.teamId,
|
||||
assetKey: embeddingKey,
|
||||
mediaType: embeddingMediaType,
|
||||
})
|
||||
results.push({ embedding: result.embedding })
|
||||
if (result.usage) {
|
||||
usage.inputTokens += result.usage.inputTokens || 0
|
||||
usage.outputTokens += result.usage.outputTokens || 0
|
||||
const alreadyEmbedded = validExistingChunks.some(
|
||||
(e) => e.startTime === null && e.endTime === null,
|
||||
)
|
||||
if (!alreadyEmbedded) {
|
||||
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
|
||||
teamId: task.teamId,
|
||||
assetKey: embeddingKey,
|
||||
mediaType: embeddingMediaType,
|
||||
})
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: [{ embedding: result.embedding }],
|
||||
})
|
||||
if (result.usage) {
|
||||
usage.inputTokens += result.usage.inputTokens || 0
|
||||
usage.outputTokens += result.usage.outputTokens || 0
|
||||
}
|
||||
}
|
||||
} else if (isVideo) {
|
||||
// Get video duration
|
||||
@@ -142,12 +188,30 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
end = duration
|
||||
}
|
||||
|
||||
// Check if this chunk is already completed
|
||||
const chunkExists = validExistingChunks.some(
|
||||
(c) =>
|
||||
c.startTime !== null &&
|
||||
c.endTime !== null &&
|
||||
Math.abs(c.startTime - start) < 0.1 &&
|
||||
Math.abs(c.endTime - end) < 0.1,
|
||||
)
|
||||
|
||||
if (chunkExists) {
|
||||
// If this chunk reached the end of the video, stop
|
||||
if (end >= duration) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Step A: Slice chunk on transcode worker and upload to S3
|
||||
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
|
||||
assetId: asset.id,
|
||||
assetKey: embeddingKey,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
format: isGemmaProvider ? 'frames' : 'video',
|
||||
})
|
||||
const { chunkKey } = chunkRes
|
||||
|
||||
@@ -162,10 +226,16 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
},
|
||||
)
|
||||
|
||||
results.push({
|
||||
embedding: chunkResult.embedding,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
// Step C: Save chunk embedding to database immediately
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: [
|
||||
{
|
||||
embedding: chunkResult.embedding,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
if (chunkResult.usage) {
|
||||
@@ -173,7 +243,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
usage.outputTokens += chunkResult.usage.outputTokens || 0
|
||||
}
|
||||
} finally {
|
||||
// Step C: Delete temporary chunk from S3
|
||||
// Step D: Delete temporary chunk from S3
|
||||
await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, {
|
||||
key: chunkKey,
|
||||
})
|
||||
@@ -186,14 +256,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
// Save computed embeddings
|
||||
if (results.length > 0) {
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: results,
|
||||
})
|
||||
}
|
||||
|
||||
// Update Usage
|
||||
await executeActivity(agentWorkerQueue, updateTaskUsageActivity, {
|
||||
taskId: task.id,
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { queryEmbeddingForSearch } from './query-embedding-for-search'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import * as workflowUtils from '@shumai/workflow-core'
|
||||
|
||||
vi.mock('@shumai/workflow-core', async () => {
|
||||
const actual = await vi.importActual('@shumai/workflow-core')
|
||||
return {
|
||||
...actual,
|
||||
getActivities: vi.fn(),
|
||||
executeActivity: vi.fn(),
|
||||
}
|
||||
})
|
||||
|
||||
describe('Query Embedding For Search Workflow', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities holds mock functions cast to expected types
|
||||
let mockActivities: any
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
|
||||
mockActivities = {
|
||||
updateWorkflowTaskActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'updateWorkflowTaskActivity',
|
||||
}),
|
||||
generateTextEmbeddingActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'generateTextEmbeddingActivity',
|
||||
}),
|
||||
getAgentWorkerQueueActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getAgentWorkerQueueActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
mockActivities.getAgentWorkerQueueActivity.mockResolvedValue('agent_queue')
|
||||
mockActivities.updateWorkflowTaskActivity.mockResolvedValue({})
|
||||
mockActivities.generateTextEmbeddingActivity.mockResolvedValue({
|
||||
embedding: [0.1, 0.2, 0.3],
|
||||
usage: { inputTokens: 4, outputTokens: 0, model: 'gemini-embedding-2' },
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities contains vi.fn mock functions which are cast to expected activity proxy types
|
||||
vi.mocked(workflowUtils.getActivities).mockReturnValue(mockActivities as any)
|
||||
vi.mocked(workflowUtils.executeActivity).mockImplementation(async (_queue, act, ...args) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- act is one of the mocked activities
|
||||
return (act as any)(...args)
|
||||
})
|
||||
})
|
||||
|
||||
it('should run query embedding workflow successfully', async () => {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'query_embedding_for_search',
|
||||
status: 'pending',
|
||||
teamId: 't1',
|
||||
assetId: 'a1',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
queryEmbeddingForSearch: {
|
||||
text: 'search query',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await queryEmbeddingForSearch(task)
|
||||
|
||||
// Verify queue discovery
|
||||
expect(mockActivities.getAgentWorkerQueueActivity).toHaveBeenCalled()
|
||||
|
||||
// Verify task processing status
|
||||
expect(mockActivities.updateWorkflowTaskActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
status: 'processing',
|
||||
heartbeat: true,
|
||||
})
|
||||
|
||||
// Verify embedding generation
|
||||
expect(mockActivities.generateTextEmbeddingActivity).toHaveBeenCalledWith({
|
||||
text: 'search query',
|
||||
teamId: 't1',
|
||||
})
|
||||
|
||||
// Verify completed status and details
|
||||
expect(mockActivities.updateWorkflowTaskActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
status: 'completed',
|
||||
output: { embedding: [0.1, 0.2, 0.3] },
|
||||
inputTokens: 4,
|
||||
outputTokens: 0,
|
||||
model: 'gemini-embedding-2',
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw immediately if text is missing', async () => {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'query_embedding_for_search',
|
||||
status: 'pending',
|
||||
teamId: 't1',
|
||||
assetId: 'a1',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(queryEmbeddingForSearch(task)).rejects.toThrow(
|
||||
'Missing text in query_embedding_for_search task payload',
|
||||
)
|
||||
|
||||
// Should not call any activities because it throws before queue discovery
|
||||
expect(mockActivities.getAgentWorkerQueueActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateWorkflowTaskActivity).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should handle activity failure and update task status to failed', async () => {
|
||||
mockActivities.generateTextEmbeddingActivity.mockRejectedValue(
|
||||
new Error('Embedding API failed'),
|
||||
)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'query_embedding_for_search',
|
||||
status: 'pending',
|
||||
teamId: 't1',
|
||||
assetId: 'a1',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
queryEmbeddingForSearch: {
|
||||
text: 'search query',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await expect(queryEmbeddingForSearch(task)).rejects.toThrow('Embedding API failed')
|
||||
|
||||
// Verify it updates task to failed
|
||||
expect(mockActivities.updateWorkflowTaskActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
status: 'failed',
|
||||
output: { error: 'Embedding API failed' },
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,56 +0,0 @@
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core'
|
||||
|
||||
export async function queryEmbeddingForSearch(task: WorkflowTask): Promise<void> {
|
||||
const { generateTextEmbeddingActivity, updateWorkflowTaskActivity, getAgentWorkerQueueActivity } =
|
||||
getActivities()
|
||||
|
||||
let agentWorkerQueue = ''
|
||||
|
||||
const payload = task.payload as PrismaJson.WorkflowTaskPayload | null
|
||||
const text = payload?.queryEmbeddingForSearch?.text
|
||||
|
||||
if (!text) {
|
||||
throw new Error('Missing text in query_embedding_for_search task payload')
|
||||
}
|
||||
|
||||
try {
|
||||
// 0. Discover queue
|
||||
agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity)
|
||||
|
||||
// 1. Mark as processing
|
||||
await executeActivity(agentWorkerQueue, updateWorkflowTaskActivity, {
|
||||
taskId: task.id,
|
||||
status: 'processing',
|
||||
heartbeat: true,
|
||||
})
|
||||
|
||||
// 2. Generate embedding
|
||||
const result = await executeActivity(agentWorkerQueue, generateTextEmbeddingActivity, {
|
||||
text,
|
||||
teamId: task.teamId!,
|
||||
})
|
||||
|
||||
// 3. Save output and complete
|
||||
await executeActivity(agentWorkerQueue, updateWorkflowTaskActivity, {
|
||||
taskId: task.id,
|
||||
status: 'completed',
|
||||
output: { embedding: result.embedding },
|
||||
inputTokens: result.usage.inputTokens,
|
||||
outputTokens: result.usage.outputTokens,
|
||||
model: result.usage.model,
|
||||
})
|
||||
} catch (err) {
|
||||
console.error(`queryEmbeddingForSearch failed for task ${task.id}:`, err)
|
||||
if (agentWorkerQueue) {
|
||||
await executeActivity(agentWorkerQueue, updateWorkflowTaskActivity, {
|
||||
taskId: task.id,
|
||||
status: 'failed',
|
||||
output: {
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
})
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
@@ -210,6 +210,30 @@ describe('Agent API', () => {
|
||||
itemId: 'agent1',
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 403 when updating embedding agent and ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
|
||||
id: 'agent1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
|
||||
const res = await app.request('/agents/agent1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Updated Embedding Agent',
|
||||
type: 'embedding',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('DELETE /agents/:agentId', () => {
|
||||
@@ -227,6 +251,25 @@ describe('Agent API', () => {
|
||||
itemId: 'agent1',
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 403 when deleting embedding agent and ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
|
||||
id: 'agent1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
|
||||
const res = await app.request('/agents/agent1', {
|
||||
method: 'DELETE',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('PATCH /agents/:agentId/permission', () => {
|
||||
@@ -271,6 +314,27 @@ describe('Agent API', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 403 when modifying embedding agent permission and ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
|
||||
id: 'agent1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
|
||||
const res = await app.request('/agents/agent1/permission', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ permission: 'owner' }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
it('denies access when user is not admin', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValue(
|
||||
new HTTPException(403, { message: 'Forbidden' }),
|
||||
@@ -410,4 +474,138 @@ describe('Agent API', () => {
|
||||
expect(agentService.listProjectChatAgents).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('read-only embedding agent when ENABLE_EMBEDDING_AGENT is true', () => {
|
||||
beforeEach(() => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
vi.mocked(agentService.getAgent).mockResolvedValue({
|
||||
id: 'emb1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
})
|
||||
|
||||
it('rejects PUT /agents/:agentId with 403', async () => {
|
||||
const res = await app.request('/agents/emb1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Updated Embedding',
|
||||
type: 'embedding',
|
||||
enabled: false,
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.updateAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects DELETE /agents/:agentId with 403', async () => {
|
||||
const res = await app.request('/agents/emb1', {
|
||||
method: 'DELETE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.deleteAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects PATCH /agents/:agentId/permission with 403', async () => {
|
||||
const res = await app.request('/agents/emb1/permission', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
permission: 'owner',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.updateAgentPermission).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects POST /teams/:teamId/agents with 403 for embedding agent', async () => {
|
||||
const res = await app.request('/teams/team1/agents', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Manual Embedding',
|
||||
type: 'embedding',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.createAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('read-only embedding agent when ENABLE_EMBEDDING_AGENT is false', () => {
|
||||
beforeEach(() => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
vi.mocked(agentService.getAgent).mockResolvedValue({
|
||||
id: 'emb1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
})
|
||||
|
||||
it('rejects PUT /agents/:agentId with 403', async () => {
|
||||
const res = await app.request('/agents/emb1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Updated Embedding',
|
||||
type: 'embedding',
|
||||
enabled: false,
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.updateAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects DELETE /agents/:agentId with 403', async () => {
|
||||
const res = await app.request('/agents/emb1', {
|
||||
method: 'DELETE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.deleteAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects PATCH /agents/:agentId/permission with 403', async () => {
|
||||
const res = await app.request('/agents/emb1/permission', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
permission: 'owner',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.updateAgentPermission).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects POST /teams/:teamId/agents with 403 for embedding agent', async () => {
|
||||
const res = await app.request('/teams/team1/agents', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Manual Embedding',
|
||||
type: 'embedding',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.createAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Hono } from 'hono'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { zValidator } from '@hono/zod-validator'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
@@ -62,6 +63,16 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const isEmbeddingManagedByEnv =
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'true' ||
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'false'
|
||||
|
||||
if (req.type === 'embedding' && isEmbeddingManagedByEnv) {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be created',
|
||||
})
|
||||
}
|
||||
|
||||
const agent = await agentService.createAgent({
|
||||
teamId,
|
||||
...req,
|
||||
@@ -92,6 +103,17 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: agentId,
|
||||
})
|
||||
|
||||
const isEmbeddingManagedByEnv =
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'true' ||
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'false'
|
||||
|
||||
const existingAgent = await agentService.getAgent({ agentId })
|
||||
if (existingAgent?.type === 'embedding' && isEmbeddingManagedByEnv) {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be modified',
|
||||
})
|
||||
}
|
||||
|
||||
const agent = await agentService.updateAgent({
|
||||
agentId,
|
||||
...req,
|
||||
@@ -123,6 +145,17 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: agentId,
|
||||
})
|
||||
|
||||
const isEmbeddingManagedByEnv =
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'true' ||
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'false'
|
||||
|
||||
const existingAgent = await agentService.getAgent({ agentId })
|
||||
if (existingAgent?.type === 'embedding' && isEmbeddingManagedByEnv) {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be modified',
|
||||
})
|
||||
}
|
||||
|
||||
const agent = await agentService.updateAgentPermission(agentId, req.permission)
|
||||
|
||||
await auditLogService.logAction({
|
||||
@@ -148,7 +181,16 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: agentId,
|
||||
})
|
||||
|
||||
const isEmbeddingManagedByEnv =
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'true' ||
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'false'
|
||||
|
||||
const existingAgent = await agentService.getAgent({ agentId })
|
||||
if (existingAgent?.type === 'embedding' && isEmbeddingManagedByEnv) {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be deleted',
|
||||
})
|
||||
}
|
||||
|
||||
await agentService.deleteAgent({
|
||||
agentId,
|
||||
|
||||
@@ -42,7 +42,8 @@ async function applyWatermarkToAssetMedia(
|
||||
password?: string,
|
||||
): Promise<AssetInfo> {
|
||||
if (!asset.media) return asset
|
||||
if (asset.proxyType !== 'video' && asset.proxyType !== 'image') return asset
|
||||
if (asset.proxyType !== 'video' && asset.proxyType !== 'image' && asset.proxyType !== '3d')
|
||||
return asset
|
||||
|
||||
const updatedMedia = { ...asset.media }
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
@@ -58,10 +59,27 @@ async function applyWatermarkToAssetMedia(
|
||||
height: vt.height ?? 0,
|
||||
size: 0,
|
||||
hdr: vt.hdr,
|
||||
resolution: vt.resolution,
|
||||
})),
|
||||
)
|
||||
: []
|
||||
|
||||
if (watermarkMedia?.videoTranscodes && watermarkMedia.videoTranscodes.length > 0) {
|
||||
if (updatedMedia.videoPreview) {
|
||||
const watermarkedKey =
|
||||
watermarkMedia.videoPreview?.key || watermarkMedia.videoTranscodes[0]?.key
|
||||
if (watermarkedKey) {
|
||||
updatedMedia.videoPreview = {
|
||||
...updatedMedia.videoPreview,
|
||||
key: watermarkedKey,
|
||||
url: await s3Service.presign(bucket, watermarkedKey, 'GET'),
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
delete updatedMedia.videoPreview
|
||||
}
|
||||
|
||||
updatedMedia.imageTranscodes =
|
||||
watermarkMedia?.imageTranscodes && watermarkMedia.imageTranscodes.length > 0
|
||||
? await Promise.all(
|
||||
@@ -254,11 +272,11 @@ const route = app
|
||||
|
||||
let asset = await assetService.getAsset({ assetId: fileId })
|
||||
|
||||
// Only video/image assets are subject to watermark protection; skip the
|
||||
// Only video/image/3d assets are subject to watermark protection; skip the
|
||||
// watermark lookup entirely for other media types (pdf, audio, ...).
|
||||
if (
|
||||
shareLink.watermarkConfigId &&
|
||||
(asset.proxyType === 'video' || asset.proxyType === 'image')
|
||||
(asset.proxyType === 'video' || asset.proxyType === 'image' || asset.proxyType === '3d')
|
||||
) {
|
||||
const targetAssetId = await assetService.resolveLatestVersionId(fileId)
|
||||
const wfMap = await watermarkService.getCompletedWatermarkMediaMap(
|
||||
|
||||
@@ -513,6 +513,121 @@ describe('Share API', () => {
|
||||
expect(body.media.hls).toBeUndefined()
|
||||
})
|
||||
|
||||
test('serves watermarked turntable video for a 3d asset when watermark enabled and completed', async () => {
|
||||
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue(shareLinkWithWatermark)
|
||||
const asset3d = {
|
||||
id: 'file1',
|
||||
name: 'model.glb',
|
||||
sizeByte: 1024,
|
||||
fileCount: 0,
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
proxyType: '3d',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoPreview: {
|
||||
url: 'http://s3/unwatermarked-turntable.mp4',
|
||||
key: 'files/turntable.mp4',
|
||||
},
|
||||
videoTranscodes: [
|
||||
{
|
||||
id: 'orig-turntable',
|
||||
url: 'http://s3/unwatermarked-turntable.mp4',
|
||||
key: 'files/turntable.mp4',
|
||||
width: 1080,
|
||||
height: 1080,
|
||||
},
|
||||
],
|
||||
original: { key: 'files/model.glb' },
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any
|
||||
vi.spyOn(assetService, 'getAsset').mockResolvedValue(asset3d)
|
||||
vi.spyOn(watermarkService, 'getCompletedWatermarkMediaMap').mockResolvedValue(
|
||||
new Map([
|
||||
[
|
||||
'file1',
|
||||
watermarkMediaInfo({
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [
|
||||
{
|
||||
key: 'files/watermarked-turntable.mp4',
|
||||
width: 1080,
|
||||
height: 1080,
|
||||
resolution: '1080p',
|
||||
},
|
||||
],
|
||||
videoPreview: { key: 'files/watermarked-turntable.mp4', width: 1080, height: 1080 },
|
||||
}),
|
||||
],
|
||||
]),
|
||||
)
|
||||
|
||||
const res = await app.request('/shares/share1/files/file1', {
|
||||
method: 'GET',
|
||||
headers: { 'x-share-password': 'pass' },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const body = await res.json()
|
||||
expect(body.media.videoTranscodes).toEqual([
|
||||
{
|
||||
id: 'files/watermarked-turntable.mp4',
|
||||
url: 'http://s3/presigned',
|
||||
key: 'files/watermarked-turntable.mp4',
|
||||
width: 1080,
|
||||
height: 1080,
|
||||
size: 0,
|
||||
resolution: '1080p',
|
||||
},
|
||||
])
|
||||
expect(body.media.videoPreview?.url).toBe('http://s3/presigned')
|
||||
expect(body.media.videoPreview?.key).toBe('files/watermarked-turntable.mp4')
|
||||
})
|
||||
|
||||
test('strips videoPreview url and serves empty videoTranscodes for a 3d asset while watermark is in flight', async () => {
|
||||
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue(shareLinkWithWatermark)
|
||||
const asset3d = {
|
||||
id: 'file1',
|
||||
name: 'model.glb',
|
||||
sizeByte: 1024,
|
||||
fileCount: 0,
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
proxyType: '3d',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
videoPreview: {
|
||||
url: 'http://s3/unwatermarked-turntable.mp4',
|
||||
key: 'files/turntable.mp4',
|
||||
},
|
||||
videoTranscodes: [
|
||||
{
|
||||
id: 'orig-turntable',
|
||||
url: 'http://s3/unwatermarked-turntable.mp4',
|
||||
key: 'files/turntable.mp4',
|
||||
width: 1080,
|
||||
height: 1080,
|
||||
},
|
||||
],
|
||||
original: { key: 'files/model.glb' },
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any
|
||||
vi.spyOn(assetService, 'getAsset').mockResolvedValue(asset3d)
|
||||
vi.spyOn(watermarkService, 'getCompletedWatermarkMediaMap').mockResolvedValue(new Map())
|
||||
|
||||
const res = await app.request('/shares/share1/files/file1', {
|
||||
method: 'GET',
|
||||
headers: { 'x-share-password': 'pass' },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const body = await res.json()
|
||||
expect(body.media.videoTranscodes).toEqual([])
|
||||
expect(body.media.videoPreview?.url).toBeUndefined()
|
||||
})
|
||||
|
||||
test('leaves non-video/image media untouched when watermark enabled', async () => {
|
||||
vi.spyOn(shareService, 'verifyPublicAccess').mockResolvedValue(shareLinkWithWatermark)
|
||||
vi.spyOn(assetService, 'getAsset').mockResolvedValue({
|
||||
|
||||
@@ -52,6 +52,22 @@ vi.mock('@shumai/core/src/auditLog/auditLog', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/turntable/turntable', () => ({
|
||||
turntableService: {
|
||||
getSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
testConnection: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/core/src/gotenberg/gotenberg', () => ({
|
||||
gotenbergService: {
|
||||
getSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
testConnection: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('team api', () => {
|
||||
const app = new Hono().use('*', authMiddleware).route('/', teamRoute)
|
||||
let mockCreateTeam: any // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
@@ -638,5 +654,208 @@ describe('team api', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/turntable-settings returns turntable settings', async () => {
|
||||
const { turntableService } = await import('@shumai/core/src/turntable/turntable')
|
||||
vi.mocked(turntableService.getSettings).mockResolvedValue({
|
||||
url: 'http://localhost:3001',
|
||||
username: 'user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/turntable-settings')
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data).toEqual({
|
||||
url: 'http://localhost:3001',
|
||||
username: 'user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('PUT /teams/:teamId/turntable-settings updates settings when not env configured', async () => {
|
||||
const { turntableService } = await import('@shumai/core/src/turntable/turntable')
|
||||
vi.mocked(turntableService.updateSettings).mockResolvedValue({
|
||||
url: 'http://custom:3001',
|
||||
username: 'newuser',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/turntable-settings', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: 'http://custom:3001',
|
||||
username: 'newuser',
|
||||
password: 'secretpassword',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.url).toBe('http://custom:3001')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('PUT /teams/:teamId/turntable-settings returns 400 when env configured', async () => {
|
||||
const { turntableService } = await import('@shumai/core/src/turntable/turntable')
|
||||
vi.mocked(turntableService.updateSettings).mockRejectedValue(
|
||||
new Error(
|
||||
'Turntable settings are configured via environment variables and cannot be edited',
|
||||
),
|
||||
)
|
||||
|
||||
const res = await app.request('/teams/t1/turntable-settings', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: 'http://custom:3001',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
const data = await res.json()
|
||||
expect(data.message).toContain('configured via environment variables')
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/turntable-settings/test tests connection', async () => {
|
||||
const { turntableService } = await import('@shumai/core/src/turntable/turntable')
|
||||
vi.mocked(turntableService.testConnection).mockResolvedValue({
|
||||
ok: true,
|
||||
version: '1.0.0',
|
||||
blender: '5.2.0',
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/turntable-settings/test', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: 'http://custom:3001',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.ok).toBe(true)
|
||||
expect(data.blender).toBe('5.2.0')
|
||||
})
|
||||
|
||||
it('GET /teams/:teamId/gotenberg-settings returns gotenberg settings', async () => {
|
||||
const { gotenbergService } = await import('@shumai/core/src/gotenberg/gotenberg')
|
||||
vi.mocked(gotenbergService.getSettings).mockResolvedValue({
|
||||
url: 'http://localhost:3000',
|
||||
username: 'user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/gotenberg-settings')
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data).toEqual({
|
||||
url: 'http://localhost:3000',
|
||||
username: 'user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('PUT /teams/:teamId/gotenberg-settings updates settings when not env configured', async () => {
|
||||
const { gotenbergService } = await import('@shumai/core/src/gotenberg/gotenberg')
|
||||
vi.mocked(gotenbergService.updateSettings).mockResolvedValue({
|
||||
url: 'http://custom:3000',
|
||||
username: 'newuser',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/gotenberg-settings', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: 'http://custom:3000',
|
||||
username: 'newuser',
|
||||
password: 'secretpassword',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.url).toBe('http://custom:3000')
|
||||
expect(authzService.hasPermission).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: 't1',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('PUT /teams/:teamId/gotenberg-settings returns 400 when env configured', async () => {
|
||||
const { gotenbergService } = await import('@shumai/core/src/gotenberg/gotenberg')
|
||||
vi.mocked(gotenbergService.updateSettings).mockRejectedValue(
|
||||
new Error(
|
||||
'Gotenberg settings are configured via environment variables and cannot be edited',
|
||||
),
|
||||
)
|
||||
|
||||
const res = await app.request('/teams/t1/gotenberg-settings', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: 'http://custom:3000',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
const data = await res.json()
|
||||
expect(data.message).toContain('configured via environment variables')
|
||||
})
|
||||
|
||||
it('POST /teams/:teamId/gotenberg-settings/test tests connection', async () => {
|
||||
const { gotenbergService } = await import('@shumai/core/src/gotenberg/gotenberg')
|
||||
vi.mocked(gotenbergService.testConnection).mockResolvedValue({
|
||||
ok: true,
|
||||
version: '8.3.0',
|
||||
})
|
||||
|
||||
const res = await app.request('/teams/t1/gotenberg-settings/test', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: 'http://custom:3000',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
const data = await res.json()
|
||||
expect(data.ok).toBe(true)
|
||||
expect(data.version).toBe('8.3.0')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -10,6 +10,8 @@ import {
|
||||
updateTeamSettingsRequestSchema,
|
||||
listMembersQuerySchema,
|
||||
updateSandboxSettingsRequestSchema,
|
||||
updateTurntableSettingsRequestSchema,
|
||||
updateGotenbergSettingsRequestSchema,
|
||||
updateMeRequestSchema,
|
||||
updateTeamMemberRoleRequestSchema,
|
||||
createApiTokenRequestSchema,
|
||||
@@ -23,6 +25,8 @@ import {
|
||||
import { apiTokenService } from '@shumai/core/src/user/api-token'
|
||||
import { aiUsageService } from '@shumai/core/src/ai-usage/ai-usage'
|
||||
import { auditLogService } from '@shumai/core/src/auditLog/auditLog'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
import { NotificationType } from '@shumai/db'
|
||||
import type { Prisma } from '@shumai/db'
|
||||
|
||||
@@ -265,6 +269,120 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
return c.json(sandbox)
|
||||
},
|
||||
)
|
||||
.get('/teams/:teamId/turntable-settings', async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const settings = await turntableService.getSettings(teamId)
|
||||
return c.json(settings)
|
||||
})
|
||||
.put(
|
||||
'/teams/:teamId/turntable-settings',
|
||||
zValidator('json', updateTurntableSettingsRequestSchema),
|
||||
async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
try {
|
||||
const settings = await turntableService.updateSettings(teamId, req)
|
||||
return c.json(settings)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
return c.json({ message }, 400)
|
||||
}
|
||||
},
|
||||
)
|
||||
.post(
|
||||
'/teams/:teamId/turntable-settings/test',
|
||||
zValidator('json', updateTurntableSettingsRequestSchema.partial()),
|
||||
async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const res = await turntableService.testConnection(req, teamId)
|
||||
return c.json(res)
|
||||
},
|
||||
)
|
||||
.get('/teams/:teamId/gotenberg-settings', async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const settings = await gotenbergService.getSettings(teamId)
|
||||
return c.json(settings)
|
||||
})
|
||||
.put(
|
||||
'/teams/:teamId/gotenberg-settings',
|
||||
zValidator('json', updateGotenbergSettingsRequestSchema),
|
||||
async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
try {
|
||||
const settings = await gotenbergService.updateSettings(teamId, req)
|
||||
return c.json(settings)
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
return c.json({ message }, 400)
|
||||
}
|
||||
},
|
||||
)
|
||||
.post(
|
||||
'/teams/:teamId/gotenberg-settings/test',
|
||||
zValidator('json', updateGotenbergSettingsRequestSchema.partial()),
|
||||
async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
const req = c.req.valid('json')
|
||||
|
||||
await authzService.hasPermission({
|
||||
user,
|
||||
permission: Permission.Admin,
|
||||
type: ResourceType.Team,
|
||||
id: teamId,
|
||||
})
|
||||
|
||||
const res = await gotenbergService.testConnection(req, teamId)
|
||||
return c.json(res)
|
||||
},
|
||||
)
|
||||
.get('/teams/:teamId/user-metadata', async (c) => {
|
||||
const user = c.get('user')
|
||||
const teamId = c.req.param('teamId')
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/core",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
@@ -29,6 +29,7 @@
|
||||
"@aws-sdk/s3-request-presigner": "3.1121.0",
|
||||
"@earendil-works/pi-agent-core": "0.82.1",
|
||||
"@earendil-works/pi-ai": "0.82.1",
|
||||
"@google/genai": "2.28.0",
|
||||
"@modelcontextprotocol/client": "2.0.0",
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
|
||||
@@ -976,4 +976,171 @@ describe('AgentService', () => {
|
||||
expect(ids).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('ensureEmbeddingAgent', () => {
|
||||
test('creates embedding agent when ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team } = await setupTestData(db)
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
|
||||
})
|
||||
expect(agent).toBeDefined()
|
||||
const info = await svc.toAgentInfo(agent!)
|
||||
expect(info.name).toBe('embedding-agent')
|
||||
expect(info.type).toBe('embedding')
|
||||
expect(info.enabled).toBe(true)
|
||||
expect(info.avatarPreset).toBe('avatar-1')
|
||||
expect(info.avatar).toContain('avatar-1.webp')
|
||||
expect(info.readOnly).toBe(true)
|
||||
|
||||
// Ensure idempotency
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const count = await db.agent.count({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
})
|
||||
expect(count).toBe(1)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
test('enables existing embedding agent when ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team } = await setupTestData(db)
|
||||
|
||||
const bot = await db.user.create({
|
||||
data: { name: 'Existing Embedding', email: 'existing-emb@shumai.ai', type: 'agent' },
|
||||
})
|
||||
await db.agent.create({
|
||||
data: {
|
||||
id: bot.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: false,
|
||||
config: { provider: '', model: '' },
|
||||
},
|
||||
})
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
|
||||
})
|
||||
expect(agent?.id).toBe(bot.id)
|
||||
expect(agent?.enabled).toBe(true)
|
||||
const info = await svc.toAgentInfo(agent!)
|
||||
expect(info.readOnly).toBe(true)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
test('does nothing when ENABLE_EMBEDDING_AGENT is not set', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const team = await db.team.create({ data: { name: 'Team No Agent' } })
|
||||
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
})
|
||||
expect(agent).toBeNull()
|
||||
})
|
||||
|
||||
test('does not create embedding agent when ENABLE_EMBEDDING_AGENT is false and agent does not exist', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team } = await setupTestData(db)
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
try {
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
})
|
||||
expect(agent).toBeNull()
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
test('disables existing embedding agent and marks readOnly: true when ENABLE_EMBEDDING_AGENT is false', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team } = await setupTestData(db)
|
||||
|
||||
const bot = await db.user.create({
|
||||
data: { name: 'Existing Embedding', email: 'existing-emb-false@shumai.ai', type: 'agent' },
|
||||
})
|
||||
await db.agent.create({
|
||||
data: {
|
||||
id: bot.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: '', model: '' },
|
||||
},
|
||||
})
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
try {
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
|
||||
})
|
||||
expect(agent?.id).toBe(bot.id)
|
||||
expect(agent?.enabled).toBe(false)
|
||||
const info = await svc.toAgentInfo(agent!)
|
||||
expect(info.readOnly).toBe(true)
|
||||
expect(info.enabled).toBe(false)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
test('ensureSystemEmbeddingAgents updates all teams when ENABLE_EMBEDDING_AGENT is false', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team: team1 } = await setupTestData(db)
|
||||
const team2 = await db.team.create({ data: { name: 'Team 2' } })
|
||||
|
||||
// Create enabled embedding agent on team1
|
||||
const bot1 = await db.user.create({
|
||||
data: { name: 'Bot 1', email: 'b1-false@shumai.ai', type: 'agent' },
|
||||
})
|
||||
await db.agent.create({
|
||||
data: {
|
||||
id: bot1.id,
|
||||
teamId: team1.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: '', model: '' },
|
||||
},
|
||||
})
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
try {
|
||||
await svc.ensureSystemEmbeddingAgents()
|
||||
|
||||
const a1 = await db.agent.findFirst({ where: { teamId: team1.id, type: 'embedding' } })
|
||||
const a2 = await db.agent.findFirst({ where: { teamId: team2.id, type: 'embedding' } })
|
||||
|
||||
expect(a1?.enabled).toBe(false)
|
||||
expect(a2).toBeNull()
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -43,8 +43,8 @@ interface AgentInfoSource {
|
||||
config: unknown
|
||||
soul: string | null
|
||||
user: { name: string; image: string | null }
|
||||
skills: Array<{ id: string; skillId: string; skill: { id: string; name: string } }>
|
||||
mcpServers: Array<{ mcpServerId: string }>
|
||||
skills?: Array<{ id: string; skillId: string; skill: { id: string; name: string } }>
|
||||
mcpServers?: Array<{ mcpServerId: string }>
|
||||
}
|
||||
|
||||
function extractS3Key(urlOrKey?: string | null): string | null | undefined {
|
||||
@@ -462,6 +462,11 @@ export class AgentService {
|
||||
*/
|
||||
async toAgentInfo(agent: AgentInfoSource): Promise<AgentInfo> {
|
||||
const config = agent.config as unknown as PrismaJson.AgentConfig
|
||||
const isManagedByEnv =
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'true' ||
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'false'
|
||||
const isReadOnly = agent.type === 'embedding' && isManagedByEnv
|
||||
|
||||
return {
|
||||
id: agent.id,
|
||||
name: agent.user.name,
|
||||
@@ -475,13 +480,14 @@ export class AgentService {
|
||||
thinkingLevel: config.thinkingLevel || 'off',
|
||||
systemPrompt: config.systemPrompt,
|
||||
soul: agent.soul || undefined,
|
||||
skills: agent.skills.map((s) => ({
|
||||
skills: (agent.skills || []).map((s) => ({
|
||||
id: s.id,
|
||||
skillId: s.skillId,
|
||||
skill: s.skill,
|
||||
})),
|
||||
mcpServerIds: (agent.mcpServers || []).map((m) => m.mcpServerId),
|
||||
deniedTools: config.deniedTools || [],
|
||||
readOnly: isReadOnly,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,6 +724,60 @@ export class AgentService {
|
||||
params,
|
||||
)
|
||||
}
|
||||
|
||||
async ensureSystemEmbeddingAgents(): Promise<void> {
|
||||
const envVal = process.env.ENABLE_EMBEDDING_AGENT
|
||||
if (envVal !== 'true' && envVal !== 'false') {
|
||||
return
|
||||
}
|
||||
|
||||
const teams = await this.prismaClient.team.findMany({
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
for (const team of teams) {
|
||||
await this.ensureEmbeddingAgentForTeam(team.id)
|
||||
}
|
||||
}
|
||||
|
||||
async ensureEmbeddingAgentForTeam(teamId: string): Promise<void> {
|
||||
const envVal = process.env.ENABLE_EMBEDDING_AGENT
|
||||
if (envVal !== 'true' && envVal !== 'false') {
|
||||
return
|
||||
}
|
||||
|
||||
const existing = await this.prismaClient.agent.findFirst({
|
||||
where: {
|
||||
teamId,
|
||||
type: 'embedding',
|
||||
},
|
||||
})
|
||||
|
||||
if (envVal === 'true') {
|
||||
if (!existing) {
|
||||
await this.createAgent({
|
||||
teamId,
|
||||
name: 'embedding-agent',
|
||||
type: 'embedding',
|
||||
avatar: 'avatar-1',
|
||||
thinkingLevel: 'off',
|
||||
enabled: true,
|
||||
})
|
||||
} else if (!existing.enabled) {
|
||||
await this.prismaClient.agent.update({
|
||||
where: { id: existing.id },
|
||||
data: { enabled: true },
|
||||
})
|
||||
}
|
||||
} else if (envVal === 'false') {
|
||||
if (existing && existing.enabled) {
|
||||
await this.prismaClient.agent.update({
|
||||
where: { id: existing.id },
|
||||
data: { enabled: false },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const agentService = new AgentService()
|
||||
|
||||
@@ -1931,6 +1931,36 @@ describe('AssetService', () => {
|
||||
expect(info.preview?.duration).toBeUndefined()
|
||||
})
|
||||
|
||||
it('toPreviewInfo handles 3d asset with proxyType 3d and omits duration', async () => {
|
||||
const { project, user } = await setupBasicAssets()
|
||||
|
||||
const model = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'model.glb',
|
||||
type: AssetType.file,
|
||||
projectId: project.id,
|
||||
creatorId: user.id,
|
||||
sizeByte: 8000,
|
||||
mediaType: 'model/gltf-binary',
|
||||
status: 'processed',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
poster: { key: 'poster-key' },
|
||||
sprite: { key: 'sprite-key', frames: 100, tileX: 10, tileY: 10 },
|
||||
metadata: { originalWidth: 1080, originalHeight: 1080, duration: 4 },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
},
|
||||
})
|
||||
|
||||
const info = await assetService.getAsset({ assetId: model.id })
|
||||
|
||||
expect(info.preview).toBeDefined()
|
||||
expect(info.preview?.proxyType).toBe('3d')
|
||||
expect(info.preview?.duration).toBeUndefined()
|
||||
expect(info.preview?.spriteUrl).toBeDefined()
|
||||
})
|
||||
|
||||
it('converts webp thumbnail to jpeg on demand when hasJpegPreview is false', async () => {
|
||||
vi.mocked(ensureJpegInStorage).mockClear()
|
||||
const { project, user } = await setupBasicAssets()
|
||||
|
||||
@@ -2407,7 +2407,13 @@ export class AssetService {
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
const proxyType = (media?.proxyType || null) as
|
||||
| 'image'
|
||||
| 'video'
|
||||
| 'audio'
|
||||
| 'pdf'
|
||||
| '3d'
|
||||
| null
|
||||
|
||||
result.push({
|
||||
id: a.id,
|
||||
@@ -2773,7 +2779,13 @@ export class AssetService {
|
||||
): Promise<PreviewInfo | null> {
|
||||
if (!asset.media) return null
|
||||
|
||||
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
const proxyType = (asset.media?.proxyType || null) as
|
||||
| 'image'
|
||||
| 'video'
|
||||
| 'audio'
|
||||
| 'pdf'
|
||||
| '3d'
|
||||
| null
|
||||
|
||||
let rawThumbKey: string | undefined
|
||||
if (proxyType === 'image' && asset.media.thumbnail?.key) {
|
||||
@@ -2826,7 +2838,7 @@ export class AssetService {
|
||||
originalHeight: asset.media.metadata?.originalHeight,
|
||||
originalWidth: asset.media.metadata?.originalWidth,
|
||||
spriteUrl,
|
||||
duration: asset.media.metadata?.duration,
|
||||
duration: proxyType === '3d' ? undefined : asset.media.metadata?.duration,
|
||||
pageCount: asset.media.metadata?.totalFrames,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,12 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as os from 'os'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { GotenbergService } from './gotenberg'
|
||||
|
||||
describe('GotenbergService', () => {
|
||||
setupTestDbHooks()
|
||||
const originalEnv = process.env
|
||||
let tmpDir: string
|
||||
let testFilePath: string
|
||||
@@ -216,4 +219,148 @@ describe('GotenbergService', () => {
|
||||
expect(indexHtmlText).toContain('{{ toHTML "file.md" }}')
|
||||
})
|
||||
})
|
||||
|
||||
describe('team settings & fallback', () => {
|
||||
it('resolves config from team settings when env is not set', async () => {
|
||||
delete process.env.GOTENBERG_URL
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Gotenberg Team',
|
||||
settings: {
|
||||
gotenberg: {
|
||||
url: 'http://custom-gotenberg:4000',
|
||||
username: 'team-user',
|
||||
password: 'team-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const service = new GotenbergService()
|
||||
const config = await service.resolveConfig(team.id)
|
||||
|
||||
expect(config).toEqual({
|
||||
url: 'http://custom-gotenberg:4000',
|
||||
username: 'team-user',
|
||||
password: 'team-password',
|
||||
isEnv: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('prioritizes environment variables over team settings', async () => {
|
||||
process.env.GOTENBERG_URL = 'http://env-gotenberg:3000'
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Gotenberg Team',
|
||||
settings: {
|
||||
gotenberg: {
|
||||
url: 'http://custom-gotenberg:4000',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const service = new GotenbergService()
|
||||
const config = await service.resolveConfig(team.id)
|
||||
|
||||
expect(config.url).toBe('http://env-gotenberg:3000')
|
||||
expect(config.isEnv).toBe(true)
|
||||
})
|
||||
|
||||
it('getSettings returns masked settings response', async () => {
|
||||
delete process.env.GOTENBERG_URL
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Gotenberg Team',
|
||||
settings: {
|
||||
gotenberg: {
|
||||
url: 'http://custom-gotenberg:4000',
|
||||
username: 'team-user',
|
||||
password: 'team-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const service = new GotenbergService()
|
||||
const settings = await service.getSettings(team.id)
|
||||
|
||||
expect(settings).toEqual({
|
||||
url: 'http://custom-gotenberg:4000',
|
||||
username: 'team-user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('updateSettings updates team settings and rejects edits when env is set', async () => {
|
||||
delete process.env.GOTENBERG_URL
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Gotenberg Team' },
|
||||
})
|
||||
|
||||
const service = new GotenbergService()
|
||||
const updated = await service.updateSettings(team.id, {
|
||||
url: 'http://new-gotenberg:3000',
|
||||
username: 'admin',
|
||||
password: 'secret-password',
|
||||
})
|
||||
|
||||
expect(updated.url).toBe('http://new-gotenberg:3000')
|
||||
expect(updated.username).toBe('admin')
|
||||
expect(updated.hasPassword).toBe(true)
|
||||
|
||||
// When env is set, reject update
|
||||
process.env.GOTENBERG_URL = 'http://env-gotenberg:3000'
|
||||
await expect(service.updateSettings(team.id, { url: 'http://another:3000' })).rejects.toThrow(
|
||||
'Gotenberg settings are configured via environment variables and cannot be edited',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('testConnection', () => {
|
||||
it('returns error when url is empty and no team settings exist', async () => {
|
||||
delete process.env.GOTENBERG_URL
|
||||
const service = new GotenbergService()
|
||||
const result = await service.testConnection({})
|
||||
expect(result.ok).toBe(false)
|
||||
expect(result.error).toBe('URL is required')
|
||||
})
|
||||
|
||||
it('successfully connects to /health and enriches with /version', async () => {
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => {
|
||||
const urlStr = String(url)
|
||||
if (urlStr.endsWith('/health')) {
|
||||
return { ok: true, status: 200 } as Response
|
||||
}
|
||||
if (urlStr.endsWith('/version')) {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ version: '8.3.0' }),
|
||||
} as Response
|
||||
}
|
||||
return { ok: false, status: 404 } as Response
|
||||
})
|
||||
|
||||
const service = new GotenbergService()
|
||||
const result = await service.testConnection({
|
||||
url: 'http://gotenberg.local:3000',
|
||||
username: 'test-user',
|
||||
password: 'test-pass',
|
||||
})
|
||||
|
||||
expect(result.ok).toBe(true)
|
||||
expect(result.version).toBe('8.3.0')
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'http://gotenberg.local:3000/health',
|
||||
expect.objectContaining({
|
||||
method: 'GET',
|
||||
headers: expect.objectContaining({
|
||||
Authorization: expect.stringMatching(/^Basic /),
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,40 +1,135 @@
|
||||
import * as fs from 'fs'
|
||||
import type { GotenbergSettingsResponse, UpdateGotenbergSettingsRequest } from '@shumai/dtos'
|
||||
import {
|
||||
IntegrationDefinition,
|
||||
resolveIntegrationConfig,
|
||||
getIntegrationSettings,
|
||||
updateIntegrationSettings,
|
||||
resolveTestConnectionConfig,
|
||||
getIntegrationBasicAuthHeader,
|
||||
ResolvedIntegrationConfig,
|
||||
} from '@shumai/core/src/integrations/helper'
|
||||
|
||||
export const GOTENBERG_INTEGRATION_DEF: IntegrationDefinition<'gotenberg'> = {
|
||||
name: 'Gotenberg',
|
||||
settingsKey: 'gotenberg',
|
||||
envKeys: {
|
||||
url: ['GOTENBERG_URL'],
|
||||
username: ['GOTENBERG_BASIC_AUTH_USERNAME'],
|
||||
password: ['GOTENBERG_BASIC_AUTH_PASSWORD'],
|
||||
},
|
||||
}
|
||||
|
||||
export interface GotenbergConvertOptions {
|
||||
landscape?: boolean
|
||||
teamId?: string
|
||||
}
|
||||
|
||||
export class GotenbergService {
|
||||
private getUrl(): string | undefined {
|
||||
const url = process.env.GOTENBERG_URL?.trim()
|
||||
return url ? url.replace(/\/+$/, '') : undefined
|
||||
async resolveConfig(teamId?: string): Promise<ResolvedIntegrationConfig> {
|
||||
return resolveIntegrationConfig(GOTENBERG_INTEGRATION_DEF, teamId)
|
||||
}
|
||||
|
||||
private getAuthHeader(): string | undefined {
|
||||
const username = process.env.GOTENBERG_BASIC_AUTH_USERNAME
|
||||
const password = process.env.GOTENBERG_BASIC_AUTH_PASSWORD
|
||||
private getAuthHeader(config: { username?: string; password?: string }): string | undefined {
|
||||
return getIntegrationBasicAuthHeader(config)
|
||||
}
|
||||
|
||||
if (username && password) {
|
||||
const credentials = Buffer.from(`${username}:${password}`).toString('base64')
|
||||
return `Basic ${credentials}`
|
||||
async getSettings(teamId: string): Promise<GotenbergSettingsResponse> {
|
||||
return getIntegrationSettings(GOTENBERG_INTEGRATION_DEF, teamId)
|
||||
}
|
||||
|
||||
async updateSettings(
|
||||
teamId: string,
|
||||
req: UpdateGotenbergSettingsRequest,
|
||||
): Promise<GotenbergSettingsResponse> {
|
||||
return updateIntegrationSettings(GOTENBERG_INTEGRATION_DEF, teamId, req)
|
||||
}
|
||||
|
||||
async testConnection(
|
||||
config?: UpdateGotenbergSettingsRequest,
|
||||
teamId?: string,
|
||||
): Promise<{ ok: boolean; version?: string; error?: string }> {
|
||||
const target = await resolveTestConnectionConfig(GOTENBERG_INTEGRATION_DEF, config, teamId)
|
||||
|
||||
if (!target.resolvedUrl) {
|
||||
return { ok: false, error: 'URL is required' }
|
||||
}
|
||||
const cleanUrl = target.resolvedUrl.replace(/\/+$/, '')
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader({
|
||||
username: target.resolvedUsername,
|
||||
password: target.resolvedPassword,
|
||||
})
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${cleanUrl}/health`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(5000),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errText = await response.text().catch(() => '')
|
||||
return {
|
||||
ok: false,
|
||||
error: `Health check returned status ${response.status}: ${errText || response.statusText}`,
|
||||
}
|
||||
}
|
||||
|
||||
// Optionally try /version to obtain version string
|
||||
let version: string | undefined
|
||||
try {
|
||||
const versionRes = await fetch(`${cleanUrl}/version`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(2000),
|
||||
})
|
||||
if (versionRes.ok) {
|
||||
const vData = (await versionRes.json().catch(() => null)) as
|
||||
| { version?: string }
|
||||
| string
|
||||
| null
|
||||
if (vData && typeof vData === 'object' && 'version' in vData) {
|
||||
version = String(vData.version)
|
||||
} else if (typeof vData === 'string') {
|
||||
version = vData
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Version endpoint is optional in some Gotenberg versions
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
version,
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
return {
|
||||
ok: false,
|
||||
error: msg,
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
const baseUrl = this.getUrl()
|
||||
if (!baseUrl) {
|
||||
async isAvailable(teamId?: string): Promise<boolean> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader()
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/health`, {
|
||||
const response = await fetch(`${config.url}/health`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(3000),
|
||||
@@ -50,9 +145,11 @@ export class GotenbergService {
|
||||
filePath: string,
|
||||
filename: string,
|
||||
options: GotenbergConvertOptions = {},
|
||||
teamId?: string,
|
||||
): Promise<Buffer> {
|
||||
const baseUrl = this.getUrl()
|
||||
if (!baseUrl) {
|
||||
const effectiveTeamId = teamId ?? options.teamId
|
||||
const config = await this.resolveConfig(effectiveTeamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Gotenberg URL is not configured.')
|
||||
}
|
||||
|
||||
@@ -65,12 +162,12 @@ export class GotenbergService {
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader()
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/forms/libreoffice/convert`, {
|
||||
const response = await fetch(`${config.url}/forms/libreoffice/convert`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: formData,
|
||||
@@ -88,9 +185,9 @@ export class GotenbergService {
|
||||
return Buffer.from(arrayBuffer)
|
||||
}
|
||||
|
||||
async convertHtmlToPdf(filePath: string, filename: string): Promise<Buffer> {
|
||||
const baseUrl = this.getUrl()
|
||||
if (!baseUrl) {
|
||||
async convertHtmlToPdf(filePath: string, filename: string, teamId?: string): Promise<Buffer> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Gotenberg URL is not configured.')
|
||||
}
|
||||
|
||||
@@ -103,12 +200,12 @@ export class GotenbergService {
|
||||
formData.append('files', new Blob([fileBytes]), targetFilename)
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader()
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/forms/chromium/convert/html`, {
|
||||
const response = await fetch(`${config.url}/forms/chromium/convert/html`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: formData,
|
||||
@@ -126,9 +223,9 @@ export class GotenbergService {
|
||||
return Buffer.from(arrayBuffer)
|
||||
}
|
||||
|
||||
async convertMarkdownToPdf(filePath: string, filename: string): Promise<Buffer> {
|
||||
const baseUrl = this.getUrl()
|
||||
if (!baseUrl) {
|
||||
async convertMarkdownToPdf(filePath: string, filename: string, teamId?: string): Promise<Buffer> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Gotenberg URL is not configured.')
|
||||
}
|
||||
|
||||
@@ -151,12 +248,12 @@ export class GotenbergService {
|
||||
formData.append('files', new Blob([fileBytes]), mdFilename)
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader()
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/forms/chromium/convert/markdown`, {
|
||||
const response = await fetch(`${config.url}/forms/chromium/convert/markdown`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: formData,
|
||||
|
||||
@@ -10,6 +10,8 @@ export * from './project/project'
|
||||
export * from './provider/provider'
|
||||
export * from './s3/s3'
|
||||
export * from './search/search'
|
||||
export * from './search/text-embedding'
|
||||
export * from './search/embedding-service'
|
||||
export * from './share/share'
|
||||
export * from './skill/skill'
|
||||
export * from './team/team'
|
||||
@@ -31,3 +33,6 @@ export * from './kanban/kanban'
|
||||
export * from './media-generation/media-generation'
|
||||
export * from './sandbox'
|
||||
export * from './asset/comments-export'
|
||||
export * from './turntable/turntable'
|
||||
export * from './utils/model3d'
|
||||
export * from './integrations/helper'
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import {
|
||||
IntegrationDefinition,
|
||||
resolveIntegrationConfig,
|
||||
getIntegrationSettings,
|
||||
updateIntegrationSettings,
|
||||
resolveTestConnectionConfig,
|
||||
getIntegrationBasicAuthHeader,
|
||||
} from './helper'
|
||||
|
||||
const TEST_INTEGRATION_DEF: IntegrationDefinition<'turntable'> = {
|
||||
name: 'TestIntegration',
|
||||
settingsKey: 'turntable',
|
||||
envKeys: {
|
||||
url: ['TEST_INTEGRATION_URL', 'FALLBACK_INTEGRATION_URL'],
|
||||
username: ['TEST_INTEGRATION_USERNAME'],
|
||||
password: ['TEST_INTEGRATION_PASSWORD'],
|
||||
},
|
||||
}
|
||||
|
||||
describe('Integration Config Helper', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
const originalEnv = process.env
|
||||
|
||||
beforeEach(() => {
|
||||
process.env = { ...originalEnv }
|
||||
delete process.env.TEST_INTEGRATION_URL
|
||||
delete process.env.FALLBACK_INTEGRATION_URL
|
||||
delete process.env.TEST_INTEGRATION_USERNAME
|
||||
delete process.env.TEST_INTEGRATION_PASSWORD
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
})
|
||||
|
||||
describe('resolveIntegrationConfig', () => {
|
||||
it('prioritizes environment variables over database settings', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://db-server:1234',
|
||||
username: 'db-user',
|
||||
password: 'db-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
process.env.TEST_INTEGRATION_URL = 'http://env-server:5678/'
|
||||
process.env.TEST_INTEGRATION_USERNAME = 'env-user'
|
||||
process.env.TEST_INTEGRATION_PASSWORD = 'env-pass'
|
||||
|
||||
const config = await resolveIntegrationConfig(TEST_INTEGRATION_DEF, team.id)
|
||||
expect(config).toEqual({
|
||||
url: 'http://env-server:5678',
|
||||
username: 'env-user',
|
||||
password: 'env-pass',
|
||||
isEnv: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to second env key if first is missing', async () => {
|
||||
process.env.FALLBACK_INTEGRATION_URL = 'http://fallback-server:8080'
|
||||
|
||||
const config = await resolveIntegrationConfig(TEST_INTEGRATION_DEF)
|
||||
expect(config.url).toBe('http://fallback-server:8080')
|
||||
expect(config.isEnv).toBe(true)
|
||||
})
|
||||
|
||||
it('reads from team settings when environment variables are not set', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://db-server:1234/',
|
||||
username: 'db-user',
|
||||
password: 'db-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const config = await resolveIntegrationConfig(TEST_INTEGRATION_DEF, team.id)
|
||||
expect(config).toEqual({
|
||||
url: 'http://db-server:1234',
|
||||
username: 'db-user',
|
||||
password: 'db-password',
|
||||
isEnv: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('returns isEnv: false and undefined fields when neither env nor DB are set', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Empty Team' },
|
||||
})
|
||||
|
||||
const config = await resolveIntegrationConfig(TEST_INTEGRATION_DEF, team.id)
|
||||
expect(config).toEqual({
|
||||
isEnv: false,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getIntegrationSettings', () => {
|
||||
it('returns safe response masking password', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://db-server:1234',
|
||||
username: 'db-user',
|
||||
password: 'db-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const res = await getIntegrationSettings(TEST_INTEGRATION_DEF, team.id)
|
||||
expect(res).toEqual({
|
||||
url: 'http://db-server:1234',
|
||||
username: 'db-user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('updateIntegrationSettings', () => {
|
||||
it('throws error when environment variable is configured', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team' },
|
||||
})
|
||||
|
||||
process.env.TEST_INTEGRATION_URL = 'http://env-server:5678'
|
||||
|
||||
await expect(
|
||||
updateIntegrationSettings(TEST_INTEGRATION_DEF, team.id, {
|
||||
url: 'http://new-url:1234',
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
'TestIntegration settings are configured via environment variables and cannot be edited',
|
||||
)
|
||||
})
|
||||
|
||||
it('updates settings and preserves existing password when not supplied', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://original:1234',
|
||||
username: 'original-user',
|
||||
password: 'secret-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const res = await updateIntegrationSettings(TEST_INTEGRATION_DEF, team.id, {
|
||||
url: 'http://updated:5678/',
|
||||
username: 'new-user',
|
||||
})
|
||||
|
||||
expect(res).toEqual({
|
||||
url: 'http://updated:5678',
|
||||
username: 'new-user',
|
||||
hasPassword: true,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
|
||||
const config = await resolveIntegrationConfig(TEST_INTEGRATION_DEF, team.id)
|
||||
expect(config.password).toBe('secret-password')
|
||||
})
|
||||
|
||||
it('clears settings in database when URL is empty or whitespace', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://to-be-cleared:1234',
|
||||
username: 'user',
|
||||
password: 'pwd',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const res = await updateIntegrationSettings(TEST_INTEGRATION_DEF, team.id, {
|
||||
url: ' ',
|
||||
})
|
||||
|
||||
expect(res).toEqual({
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
})
|
||||
|
||||
const updatedTeam = await prisma.team.findUnique({
|
||||
where: { id: team.id },
|
||||
})
|
||||
// Check that turntable is removed from team settings
|
||||
expect((updatedTeam?.settings as Record<string, unknown>)?.turntable).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveTestConnectionConfig', () => {
|
||||
it('uses provided credentials directly when supplied', async () => {
|
||||
const target = await resolveTestConnectionConfig(TEST_INTEGRATION_DEF, {
|
||||
url: 'http://direct:1234',
|
||||
username: 'direct-user',
|
||||
password: 'direct-password',
|
||||
})
|
||||
|
||||
expect(target).toEqual({
|
||||
resolvedUrl: 'http://direct:1234',
|
||||
resolvedUsername: 'direct-user',
|
||||
resolvedPassword: 'direct-password',
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to saved database settings when testing with partial inputs', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://db:1234',
|
||||
username: 'db-user',
|
||||
password: 'db-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const target = await resolveTestConnectionConfig(
|
||||
TEST_INTEGRATION_DEF,
|
||||
{ url: 'http://new-url:9999' },
|
||||
team.id,
|
||||
)
|
||||
|
||||
expect(target).toEqual({
|
||||
resolvedUrl: 'http://new-url:9999',
|
||||
resolvedUsername: 'db-user',
|
||||
resolvedPassword: 'db-password',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getIntegrationBasicAuthHeader', () => {
|
||||
it('generates base64 basic auth header when username and password are provided', () => {
|
||||
const header = getIntegrationBasicAuthHeader({ username: 'admin', password: 'secret' })
|
||||
expect(header).toBe(`Basic ${Buffer.from('admin:secret').toString('base64')}`)
|
||||
})
|
||||
|
||||
it('returns undefined when username or password is missing', () => {
|
||||
expect(getIntegrationBasicAuthHeader({ username: 'admin' })).toBeUndefined()
|
||||
expect(getIntegrationBasicAuthHeader({ password: 'secret' })).toBeUndefined()
|
||||
expect(getIntegrationBasicAuthHeader({})).toBeUndefined()
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,188 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import '@shumai/db/src/prisma-json-types'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
export interface IntegrationEnvKeys {
|
||||
url: string[]
|
||||
username?: string[]
|
||||
password?: string[]
|
||||
}
|
||||
|
||||
export interface IntegrationDefinition<TKey extends keyof PrismaJson.Settings> {
|
||||
name: string
|
||||
settingsKey: TKey
|
||||
envKeys: IntegrationEnvKeys
|
||||
}
|
||||
|
||||
export interface ResolvedIntegrationConfig {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
isEnv: boolean
|
||||
}
|
||||
|
||||
export interface IntegrationSettingsResponse {
|
||||
url?: string
|
||||
username?: string
|
||||
hasPassword: boolean
|
||||
isEnvConfigured: boolean
|
||||
}
|
||||
|
||||
export interface UpdateIntegrationSettingsRequest {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
}
|
||||
|
||||
function getFirstEnvValue(keys?: string[]): string | undefined {
|
||||
if (!keys) return undefined
|
||||
for (const key of keys) {
|
||||
const val = process.env[key]?.trim()
|
||||
if (val) return val
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export async function resolveIntegrationConfig<TKey extends keyof PrismaJson.Settings>(
|
||||
def: IntegrationDefinition<TKey>,
|
||||
teamId?: string,
|
||||
): Promise<ResolvedIntegrationConfig> {
|
||||
const envUrl = getFirstEnvValue(def.envKeys.url)
|
||||
const envUsername = getFirstEnvValue(def.envKeys.username)
|
||||
const envPassword = getFirstEnvValue(def.envKeys.password)
|
||||
|
||||
if (envUrl) {
|
||||
return {
|
||||
url: envUrl.replace(/\/+$/, ''),
|
||||
username: envUsername || undefined,
|
||||
password: envPassword || undefined,
|
||||
isEnv: true,
|
||||
}
|
||||
}
|
||||
|
||||
if (teamId) {
|
||||
try {
|
||||
const team = await prisma.team.findUnique({
|
||||
where: { id: teamId },
|
||||
select: { settings: true },
|
||||
})
|
||||
const teamSettings = team?.settings as PrismaJson.Settings | null
|
||||
const serviceSettings = teamSettings?.[def.settingsKey] as
|
||||
| { url?: string; username?: string; password?: string }
|
||||
| undefined
|
||||
|
||||
if (serviceSettings?.url?.trim()) {
|
||||
return {
|
||||
url: serviceSettings.url.trim().replace(/\/+$/, ''),
|
||||
username: serviceSettings.username?.trim() || undefined,
|
||||
password: serviceSettings.password?.trim() || undefined,
|
||||
isEnv: false,
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error({ err, teamId }, `Failed to load team ${def.name} settings`)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
isEnv: false,
|
||||
}
|
||||
}
|
||||
|
||||
export async function getIntegrationSettings<TKey extends keyof PrismaJson.Settings>(
|
||||
def: IntegrationDefinition<TKey>,
|
||||
teamId: string,
|
||||
): Promise<IntegrationSettingsResponse> {
|
||||
const config = await resolveIntegrationConfig(def, teamId)
|
||||
return {
|
||||
url: config.url,
|
||||
username: config.username,
|
||||
hasPassword: Boolean(config.password),
|
||||
isEnvConfigured: config.isEnv,
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateIntegrationSettings<TKey extends keyof PrismaJson.Settings>(
|
||||
def: IntegrationDefinition<TKey>,
|
||||
teamId: string,
|
||||
req: UpdateIntegrationSettingsRequest,
|
||||
): Promise<IntegrationSettingsResponse> {
|
||||
const currentConfig = await resolveIntegrationConfig(def, teamId)
|
||||
if (currentConfig.isEnv) {
|
||||
throw new Error(
|
||||
`${def.name} settings are configured via environment variables and cannot be edited`,
|
||||
)
|
||||
}
|
||||
|
||||
const team = await prisma.team.findUnique({
|
||||
where: { id: teamId },
|
||||
select: { settings: true },
|
||||
})
|
||||
const settings = { ...((team?.settings || {}) as PrismaJson.Settings) }
|
||||
const prevSettings = (settings[def.settingsKey] || {}) as {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
}
|
||||
|
||||
const trimmedUrl = req.url?.trim()
|
||||
if (!trimmedUrl) {
|
||||
// If URL is empty/unset, clear the integration settings
|
||||
delete settings[def.settingsKey]
|
||||
} else {
|
||||
const updatedServiceSettings: { url: string; username?: string; password?: string } = {
|
||||
url: trimmedUrl.replace(/\/+$/, ''),
|
||||
username:
|
||||
req.username !== undefined ? req.username.trim() || undefined : prevSettings.username,
|
||||
password: req.password !== undefined ? req.password : prevSettings.password,
|
||||
}
|
||||
settings[def.settingsKey] = updatedServiceSettings as unknown as PrismaJson.Settings[TKey]
|
||||
}
|
||||
|
||||
await prisma.team.update({
|
||||
where: { id: teamId },
|
||||
data: { settings },
|
||||
})
|
||||
|
||||
return getIntegrationSettings(def, teamId)
|
||||
}
|
||||
|
||||
export async function resolveTestConnectionConfig<TKey extends keyof PrismaJson.Settings>(
|
||||
def: IntegrationDefinition<TKey>,
|
||||
config?: UpdateIntegrationSettingsRequest,
|
||||
teamId?: string,
|
||||
): Promise<{ resolvedUrl?: string; resolvedUsername?: string; resolvedPassword?: string }> {
|
||||
let resolvedUrl = config?.url?.trim()
|
||||
let resolvedUsername = config?.username?.trim()
|
||||
let resolvedPassword = config?.password
|
||||
|
||||
if (teamId && (!resolvedUrl || resolvedPassword === undefined)) {
|
||||
const savedConfig = await resolveIntegrationConfig(def, teamId)
|
||||
if (!resolvedUrl) {
|
||||
resolvedUrl = savedConfig.url
|
||||
}
|
||||
if (!resolvedUsername && savedConfig.username) {
|
||||
resolvedUsername = savedConfig.username
|
||||
}
|
||||
if (resolvedPassword === undefined) {
|
||||
resolvedPassword = savedConfig.password
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
resolvedUrl,
|
||||
resolvedUsername,
|
||||
resolvedPassword,
|
||||
}
|
||||
}
|
||||
|
||||
export function getIntegrationBasicAuthHeader(config: {
|
||||
username?: string
|
||||
password?: string
|
||||
}): string | undefined {
|
||||
if (config.username && config.password) {
|
||||
const credentials = Buffer.from(`${config.username}:${config.password}`).toString('base64')
|
||||
return `Basic ${credentials}`
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { prisma, AssetType, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { embeddingService } from './embedding-service'
|
||||
|
||||
describe('EmbeddingService', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
it('should skip teams without an enabled embedding agent', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team Without Agent' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project 1', teamId: team.id },
|
||||
})
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
const result = await embeddingService.regenerateEmbeddings()
|
||||
expect(result.tasksCreated).toBe(0)
|
||||
expect(result.totalAssets).toBe(0)
|
||||
expect(result.skippedTeams).toContain('Team Without Agent')
|
||||
})
|
||||
|
||||
it('should skip teams where embedding agent exists but is disabled', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Team Disabled Agent' },
|
||||
})
|
||||
await agentService.createAgent({
|
||||
teamId: team.id,
|
||||
name: 'embedding-agent',
|
||||
type: 'embedding',
|
||||
avatar: 'avatar-1',
|
||||
thinkingLevel: 'off',
|
||||
enabled: false,
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Project Disabled', teamId: team.id },
|
||||
})
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'photo.jpg',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'image/jpeg',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
const result = await embeddingService.regenerateEmbeddings()
|
||||
expect(result.tasksCreated).toBe(0)
|
||||
expect(result.totalAssets).toBe(0)
|
||||
expect(result.skippedTeams).toContain('Team Disabled Agent')
|
||||
})
|
||||
|
||||
it('should regenerate embeddings for eligible media assets and clean up old data', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Active Embedding Team' },
|
||||
})
|
||||
const agent = await agentService.createAgent({
|
||||
teamId: team.id,
|
||||
name: 'embedding-agent',
|
||||
type: 'embedding',
|
||||
avatar: 'avatar-1',
|
||||
thinkingLevel: 'off',
|
||||
enabled: true,
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Active Project', teamId: team.id },
|
||||
})
|
||||
|
||||
// 1. Eligible video asset
|
||||
const videoAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'sample.mp4',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Eligible image asset
|
||||
const imageAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'sample.png',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'image/png',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 3. Eligible 3D asset with rendered views
|
||||
const asset3d = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'model.glb',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'model/gltf-binary',
|
||||
media: {
|
||||
proxyType: '3d',
|
||||
imageTranscodes: [
|
||||
{ key: 'files/3d/view.webp', format: 'webp', width: 100, height: 100, quality: 80 },
|
||||
],
|
||||
} as unknown as PrismaJson.MediaInfo,
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 4. Ineligible asset (non-media document)
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'document.pdf',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'application/pdf',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 5. Ineligible asset (trashed / deleted)
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'deleted-video.mp4',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
isDeleted: true,
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// 6. Ineligible asset (status not processed)
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'uploading.mp4',
|
||||
type: AssetType.file,
|
||||
status: 'uploading',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Create preexisting embeddings and workflow task for videoAsset to verify cleanup
|
||||
const dummyVec768 = `[${new Array(768).fill(0.1).join(',')}]`
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${videoAsset.id}, ${dummyVec768}::vector, NOW())
|
||||
`
|
||||
await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: videoAsset.id,
|
||||
type: WorkflowTaskType.ai_embedding,
|
||||
status: WorkflowTaskStatus.processing,
|
||||
teamId: team.id,
|
||||
projectId: project.id,
|
||||
},
|
||||
})
|
||||
|
||||
const initialEmbeddings = await prisma.assetEmbedding.count({
|
||||
where: { assetId: videoAsset.id },
|
||||
})
|
||||
expect(initialEmbeddings).toBe(1)
|
||||
|
||||
// Run regeneration
|
||||
const result = await embeddingService.regenerateEmbeddings()
|
||||
|
||||
expect(result.totalAssets).toBe(3)
|
||||
expect(result.tasksCreated).toBe(3)
|
||||
expect(result.skippedTeams).not.toContain('Active Embedding Team')
|
||||
|
||||
// Verify old embeddings were wiped upfront
|
||||
const remainingEmbeddings = await prisma.assetEmbedding.count({
|
||||
where: { assetId: videoAsset.id },
|
||||
})
|
||||
expect(remainingEmbeddings).toBe(0)
|
||||
|
||||
// Verify tasks created
|
||||
const createdTasks = await prisma.workflowTask.findMany({
|
||||
where: {
|
||||
type: WorkflowTaskType.ai_embedding,
|
||||
assetId: { in: [videoAsset.id, imageAsset.id, asset3d.id] },
|
||||
},
|
||||
})
|
||||
expect(createdTasks).toHaveLength(3)
|
||||
for (const task of createdTasks) {
|
||||
expect(task.status).toBe(WorkflowTaskStatus.pending)
|
||||
expect(task.teamId).toBe(team.id)
|
||||
expect(task.projectId).toBe(project.id)
|
||||
const payload = task.payload as { agent?: { agentId?: string } }
|
||||
expect(payload?.agent?.agentId).toBe(agent!.id)
|
||||
}
|
||||
})
|
||||
|
||||
it('should handle multiple teams where only some have active embedding agents', async () => {
|
||||
// Team 1: has active agent
|
||||
const team1 = await prisma.team.create({ data: { name: 'Team One' } })
|
||||
await agentService.createAgent({
|
||||
teamId: team1.id,
|
||||
name: 'embedding-agent',
|
||||
type: 'embedding',
|
||||
avatar: 'avatar-1',
|
||||
thinkingLevel: 'off',
|
||||
enabled: true,
|
||||
})
|
||||
const project1 = await prisma.project.create({ data: { name: 'P1', teamId: team1.id } })
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'v1.mp4',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project1.id,
|
||||
},
|
||||
})
|
||||
|
||||
// Team 2: no active agent
|
||||
const team2 = await prisma.team.create({ data: { name: 'Team Two' } })
|
||||
const project2 = await prisma.project.create({ data: { name: 'P2', teamId: team2.id } })
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'v2.mp4',
|
||||
type: AssetType.file,
|
||||
status: 'processed',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project2.id,
|
||||
},
|
||||
})
|
||||
|
||||
const result = await embeddingService.regenerateEmbeddings()
|
||||
|
||||
expect(result.totalAssets).toBe(1)
|
||||
expect(result.tasksCreated).toBe(1)
|
||||
expect(result.skippedTeams).toContain('Team Two')
|
||||
expect(result.skippedTeams).not.toContain('Team One')
|
||||
|
||||
const tasks = await prisma.workflowTask.findMany({
|
||||
where: {
|
||||
type: WorkflowTaskType.ai_embedding,
|
||||
teamId: { in: [team1.id, team2.id] },
|
||||
},
|
||||
})
|
||||
expect(tasks).toHaveLength(1)
|
||||
expect(tasks[0].teamId).toBe(team1.id)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,135 @@
|
||||
import { prisma, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
export interface RegenerateEmbeddingsResult {
|
||||
tasksCreated: number
|
||||
totalAssets: number
|
||||
skippedTeams: string[]
|
||||
}
|
||||
|
||||
export class EmbeddingService {
|
||||
/**
|
||||
* Regenerates vector embeddings across all teams.
|
||||
*
|
||||
* Skips teams that do not have an active (enabled) embedding agent and logs a warning.
|
||||
* For teams with an active embedding agent, purges existing embeddings and pending/processing
|
||||
* embedding tasks upfront, then creates fresh ai_embedding workflow tasks for all eligible assets.
|
||||
*/
|
||||
async regenerateEmbeddings(): Promise<RegenerateEmbeddingsResult> {
|
||||
const teams = await prisma.team.findMany({
|
||||
select: { id: true, name: true },
|
||||
})
|
||||
|
||||
const activeAgents = await prisma.agent.findMany({
|
||||
where: {
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
},
|
||||
select: { id: true, teamId: true },
|
||||
})
|
||||
|
||||
const activeAgentByTeamId = new Map(
|
||||
activeAgents
|
||||
.filter((a): a is typeof a & { teamId: string } => typeof a.teamId === 'string')
|
||||
.map((a) => [a.teamId, a]),
|
||||
)
|
||||
|
||||
const activeTeamIds: string[] = []
|
||||
const skippedTeams: string[] = []
|
||||
|
||||
for (const team of teams) {
|
||||
if (activeAgentByTeamId.has(team.id)) {
|
||||
activeTeamIds.push(team.id)
|
||||
} else {
|
||||
skippedTeams.push(team.name || team.id)
|
||||
logger.warn(
|
||||
{ teamId: team.id, teamName: team.name },
|
||||
'Skipping embedding regeneration for team: no active embedding agent found',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (activeTeamIds.length === 0) {
|
||||
logger.info('No teams found with an active embedding agent. 0 tasks created.')
|
||||
return { tasksCreated: 0, totalAssets: 0, skippedTeams }
|
||||
}
|
||||
|
||||
const candidateAssets = await prisma.asset.findMany({
|
||||
where: {
|
||||
isDeleted: false,
|
||||
status: 'processed',
|
||||
type: { in: ['file', 'version_stack'] },
|
||||
project: { teamId: { in: activeTeamIds } },
|
||||
},
|
||||
include: { project: true },
|
||||
})
|
||||
|
||||
const eligibleAssets = candidateAssets.filter((asset) => {
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
|
||||
const isImage =
|
||||
asset.mediaType?.startsWith('image/') ||
|
||||
(mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0)
|
||||
return isVideo || isImage
|
||||
})
|
||||
|
||||
const eligibleAssetIds = eligibleAssets.map((a) => a.id)
|
||||
|
||||
if (eligibleAssetIds.length > 0) {
|
||||
// Clean up previous embedding tasks to avoid duplicate execution
|
||||
await prisma.workflowTask.deleteMany({
|
||||
where: {
|
||||
assetId: { in: eligibleAssetIds },
|
||||
type: WorkflowTaskType.ai_embedding,
|
||||
},
|
||||
})
|
||||
|
||||
// Purge existing embeddings so everything regenerates cleanly with the configured provider
|
||||
await prisma.assetEmbedding.deleteMany({
|
||||
where: {
|
||||
assetId: { in: eligibleAssetIds },
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
let tasksCreated = 0
|
||||
for (const asset of eligibleAssets) {
|
||||
const teamId = asset.project?.teamId
|
||||
if (!teamId) continue
|
||||
const agent = activeAgentByTeamId.get(teamId)
|
||||
if (!agent) continue
|
||||
|
||||
await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
type: WorkflowTaskType.ai_embedding,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
teamId,
|
||||
projectId: asset.projectId,
|
||||
payload: {
|
||||
projectId: asset.projectId ?? '',
|
||||
agent: { agentId: agent.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
tasksCreated++
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
totalAssets: eligibleAssets.length,
|
||||
tasksCreated,
|
||||
skippedTeamsCount: skippedTeams.length,
|
||||
},
|
||||
'Successfully queued embedding regeneration tasks',
|
||||
)
|
||||
|
||||
return {
|
||||
tasksCreated,
|
||||
totalAssets: eligibleAssets.length,
|
||||
skippedTeams,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const embeddingService = new EmbeddingService()
|
||||
@@ -4,7 +4,7 @@ import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { SearchService } from './search'
|
||||
import { SearchConditionOperator } from '@shumai/dtos'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { textEmbeddingService } from './text-embedding'
|
||||
|
||||
import { AssetType } from '@shumai/db'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
@@ -805,20 +805,14 @@ describe('SearchService', () => {
|
||||
)
|
||||
`
|
||||
|
||||
// Mock the embedding generation workflow task.
|
||||
// Mock the embedding generation
|
||||
// Our query vector points along the first dimension [1, 0, 0...]
|
||||
const queryVector = Array(1536).fill(0)
|
||||
queryVector[0] = 1.0
|
||||
const mockExecuteWait = vi
|
||||
.spyOn(workflowService, 'executeWait')
|
||||
.mockImplementation(async (task) => {
|
||||
return {
|
||||
...task,
|
||||
status: 'completed',
|
||||
output: {
|
||||
embedding: queryVector,
|
||||
},
|
||||
} as unknown as typeof task
|
||||
const mockGenerateEmbedding = vi
|
||||
.spyOn(textEmbeddingService, 'generateEmbedding')
|
||||
.mockResolvedValue({
|
||||
embedding: queryVector,
|
||||
})
|
||||
|
||||
// Search using semantic mode
|
||||
@@ -855,7 +849,248 @@ describe('SearchService', () => {
|
||||
expect(result.data[2].startTime).toBe(10)
|
||||
expect(result.data[2].endTime).toBe(20)
|
||||
|
||||
mockExecuteWait.mockRestore()
|
||||
mockGenerateEmbedding.mockRestore()
|
||||
})
|
||||
|
||||
it('matches and sorts 768-dimensional embeddings using embedding_768 column', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'semantic-user-768', email: 'semantic-user-768@example.com', type: 'human' },
|
||||
})
|
||||
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'semantic-team-768' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'semantic-project-768', teamId: team.id },
|
||||
})
|
||||
|
||||
const botUser = await prisma.user.create({
|
||||
data: { name: 'Embedding Bot 768', email: 'emb-768@example.com', type: 'agent' },
|
||||
})
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'gemma',
|
||||
teamId: team.id,
|
||||
config: { api: 'openai', apiKey: 'key' } as unknown as PrismaJson.ProviderConfig,
|
||||
},
|
||||
})
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'embeddinggemma-2',
|
||||
name: 'EmbeddingGemma 2',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 1000,
|
||||
maxTokens: 1000,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
} as unknown as PrismaJson.ModelConfig,
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: botUser.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: { provider: 'gemma-embedding-2-openai', model: 'embeddinggemma-2' },
|
||||
},
|
||||
})
|
||||
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root-768',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const videoAssetA = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'VideoA_768.mp4',
|
||||
type: AssetType.file,
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const videoAssetB = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'VideoB_768.mp4',
|
||||
type: AssetType.file,
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const arrA1 = Array(768).fill(0)
|
||||
arrA1[0] = 1.0
|
||||
const vectorA1 = JSON.stringify(arrA1)
|
||||
|
||||
const arrB1 = Array(768).fill(0)
|
||||
arrB1[0] = 1.0
|
||||
arrB1[1] = 1.0
|
||||
const vectorB1 = JSON.stringify(arrB1)
|
||||
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, created_at, updated_at)
|
||||
VALUES (
|
||||
'chunk-768-a1',
|
||||
${videoAssetA.id},
|
||||
${vectorA1}::vector,
|
||||
0.0,
|
||||
10.0,
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
`
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, created_at, updated_at)
|
||||
VALUES (
|
||||
'chunk-768-b1',
|
||||
${videoAssetB.id},
|
||||
${vectorB1}::vector,
|
||||
5.0,
|
||||
15.0,
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
`
|
||||
|
||||
// Query vector with 768 dims pointing along dimension 0
|
||||
const queryVector = Array(768).fill(0)
|
||||
queryVector[0] = 1.0
|
||||
const mockGenerateEmbedding = vi
|
||||
.spyOn(textEmbeddingService, 'generateEmbedding')
|
||||
.mockResolvedValue({
|
||||
embedding: queryVector,
|
||||
})
|
||||
|
||||
const result = await searchService.search(rootFolder.id, {
|
||||
recursively: true,
|
||||
assetType: 'file',
|
||||
operator: 'AND',
|
||||
conditions: [],
|
||||
query: 'find 768 dim match',
|
||||
isSemantic: true,
|
||||
})
|
||||
|
||||
expect(result.data.length).toBeGreaterThanOrEqual(2)
|
||||
expect(result.data[0].id).toBe(videoAssetA.id)
|
||||
expect(result.data[1].id).toBe(videoAssetB.id)
|
||||
|
||||
mockGenerateEmbedding.mockRestore()
|
||||
})
|
||||
|
||||
it('throws 502 if embedding generation fails during semantic search', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'semantic-user-fail', email: 'semantic-fail@example.com', type: 'human' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'semantic-team-fail' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'semantic-project-fail', teamId: team.id },
|
||||
})
|
||||
const botUser = await prisma.user.create({
|
||||
data: { name: 'Embedding Bot Fail', email: 'emb-fail@example.com', type: 'agent' },
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: botUser.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: 'gemini-embedding-2', model: 'gemini-embedding-2' },
|
||||
},
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root-fail',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const mockGenerateEmbedding = vi
|
||||
.spyOn(textEmbeddingService, 'generateEmbedding')
|
||||
.mockRejectedValue(new Error('Rate limit exceeded'))
|
||||
|
||||
await expect(
|
||||
searchService.search(rootFolder.id, {
|
||||
recursively: true,
|
||||
assetType: 'file',
|
||||
operator: 'AND',
|
||||
conditions: [],
|
||||
query: 'fail query',
|
||||
isSemantic: true,
|
||||
}),
|
||||
).rejects.toThrow('Failed to generate query embedding: Rate limit exceeded')
|
||||
|
||||
mockGenerateEmbedding.mockRestore()
|
||||
})
|
||||
|
||||
it('rejects semantic search when ENABLE_EMBEDDING_AGENT is false', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'u-emb-false', email: 'u-emb-false@example.com', type: 'human' },
|
||||
})
|
||||
const team = await prisma.team.create({ data: { name: 't-emb-false' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'p-emb-false', teamId: team.id },
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root-false',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
// Even if an embedding agent is enabled in db
|
||||
const bot = await prisma.user.create({
|
||||
data: { name: 'Bot', email: 'b-emb-false@example.com', password: 'pw', type: 'agent' },
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: bot.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: '', model: '' },
|
||||
},
|
||||
})
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
try {
|
||||
await expect(
|
||||
searchService.search(rootFolder.id, {
|
||||
recursively: true,
|
||||
assetType: 'file',
|
||||
operator: 'AND',
|
||||
conditions: [],
|
||||
query: 'test query',
|
||||
isSemantic: true,
|
||||
}),
|
||||
).rejects.toThrow('Embedding agent not configured or disabled for this team.')
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { Prisma, AssetType, WorkflowTaskType } from '@shumai/db'
|
||||
import { Prisma, AssetType } from '@shumai/db'
|
||||
import { AssetService, assetService } from '@shumai/core/src/asset/asset'
|
||||
import { AssetInfo } from '@shumai/dtos'
|
||||
import { SearchRequest } from '@shumai/dtos'
|
||||
import { PaginatedData, decodeCursor, encodeCursor, PageInfo } from '@shumai/core/src/pagination'
|
||||
import { generateSearchNgrams } from '@shumai/core/src/utils/ngram'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { SqlQueryBuilder } from './sql-query-builder'
|
||||
import { TextEmbeddingService, textEmbeddingService } from './text-embedding'
|
||||
|
||||
export class SearchService {
|
||||
constructor(
|
||||
private readonly prismaClient: typeof prisma = prisma,
|
||||
private readonly assetSvc: AssetService = assetService,
|
||||
private readonly embeddingSvc: TextEmbeddingService = textEmbeddingService,
|
||||
) {}
|
||||
|
||||
async search(folderId: string, req: SearchRequest): Promise<PaginatedData<AssetInfo[]>> {
|
||||
@@ -34,6 +35,12 @@ export class SearchService {
|
||||
const teamId = team?.project?.teamId
|
||||
if (!teamId) throw new Error('Team ID not found for folder')
|
||||
|
||||
if (process.env.ENABLE_EMBEDDING_AGENT === 'false') {
|
||||
throw new HTTPException(422, {
|
||||
message: 'Embedding agent not configured or disabled for this team.',
|
||||
})
|
||||
}
|
||||
|
||||
const embeddingAgent = await this.prismaClient.agent.findFirst({
|
||||
where: { teamId, type: 'embedding', enabled: true },
|
||||
})
|
||||
@@ -44,36 +51,43 @@ export class SearchService {
|
||||
})
|
||||
}
|
||||
|
||||
// 2. Generate query embedding via workflow
|
||||
const task = await this.prismaClient.workflowTask.create({
|
||||
data: {
|
||||
type: WorkflowTaskType.query_embedding_for_search,
|
||||
// 2. Generate query embedding directly using configured environment provider
|
||||
let queryVector: number[]
|
||||
try {
|
||||
const result = await this.embeddingSvc.generateEmbedding(req.query, {
|
||||
teamId,
|
||||
assetId: folderId,
|
||||
payload: {
|
||||
projectId: '',
|
||||
queryEmbeddingForSearch: { text: req.query },
|
||||
} as PrismaJson.WorkflowTaskPayload,
|
||||
status: 'pending',
|
||||
},
|
||||
})
|
||||
})
|
||||
queryVector = result.embedding
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
throw new HTTPException(502, {
|
||||
message: `Failed to generate query embedding: ${msg}`,
|
||||
})
|
||||
}
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
const output = completedTask.output as Record<string, unknown> | null
|
||||
const queryVector = output?.embedding as number[] | undefined
|
||||
|
||||
if (!queryVector) {
|
||||
throw new Error('Failed to generate query embedding')
|
||||
if (!queryVector || queryVector.length === 0) {
|
||||
throw new HTTPException(502, {
|
||||
message: 'Failed to generate query embedding: empty vector returned',
|
||||
})
|
||||
}
|
||||
|
||||
// 3. Construct raw SQL using SqlQueryBuilder
|
||||
const vectorJson = JSON.stringify(queryVector)
|
||||
const is768 = queryVector.length === 768
|
||||
const distanceExpr = is768
|
||||
? Prisma.sql`(ae.embedding_768 <=> ${vectorJson}::vector)`
|
||||
: Prisma.sql`(ae.embedding <=> ${vectorJson}::vector)`
|
||||
const notNullCond = is768
|
||||
? Prisma.sql`ae.embedding_768 IS NOT NULL`
|
||||
: Prisma.sql`ae.embedding IS NOT NULL`
|
||||
|
||||
const builder = new SqlQueryBuilder()
|
||||
.select(
|
||||
Prisma.sql`a.id as "assetId", ae.start_time as "startTime", ae.end_time as "endTime", (ae.embedding <=> ${vectorJson}::vector) as "distance"`,
|
||||
Prisma.sql`a.id as "assetId", ae.start_time as "startTime", ae.end_time as "endTime", ${distanceExpr} as "distance"`,
|
||||
)
|
||||
.from(Prisma.sql`assets a JOIN asset_embeddings ae ON a.id = ae.asset_id`)
|
||||
.addWhere(Prisma.sql`a.is_deleted = false`)
|
||||
.addWhere(notNullCond)
|
||||
|
||||
if (targetFolderIds.length > 0) {
|
||||
builder.addWhere(Prisma.sql`a.parent_id = ANY(${targetFolderIds})`)
|
||||
@@ -158,6 +172,7 @@ export class SearchService {
|
||||
.select(Prisma.sql`COUNT(*)`)
|
||||
.from(Prisma.sql`assets a JOIN asset_embeddings ae ON a.id = ae.asset_id`)
|
||||
.addWhere(Prisma.sql`a.is_deleted = false`)
|
||||
.addWhere(notNullCond)
|
||||
|
||||
if (targetFolderIds.length > 0) {
|
||||
countBuilder.addWhere(Prisma.sql`a.parent_id = ANY(${targetFolderIds})`)
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { TextEmbeddingService } from './text-embedding'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
const mockEmbedContent = vi.fn()
|
||||
|
||||
vi.mock('@google/genai', () => {
|
||||
return {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
GoogleGenAI: class {
|
||||
models = {
|
||||
embedContent: mockEmbedContent,
|
||||
}
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
describe('TextEmbeddingService', () => {
|
||||
const originalEnv = { ...process.env }
|
||||
let service: TextEmbeddingService
|
||||
|
||||
beforeEach(() => {
|
||||
process.env = { ...originalEnv }
|
||||
process.env.GEMINI_API_KEY = 'test-gemini-key'
|
||||
delete process.env.EMBEDDING_PROVIDER
|
||||
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
|
||||
mockEmbedContent.mockReset()
|
||||
service = new TextEmbeddingService(500, 3600 * 1000)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('throws an error if text is empty or whitespace', async () => {
|
||||
await expect(service.generateEmbedding('')).rejects.toThrow('Text to embed must not be empty')
|
||||
await expect(service.generateEmbedding(' ')).rejects.toThrow(
|
||||
'Text to embed must not be empty',
|
||||
)
|
||||
})
|
||||
|
||||
it('generates Gemini embedding (1536d) by default', async () => {
|
||||
const dummyEmbedding = Array(1536).fill(0.05)
|
||||
mockEmbedContent.mockResolvedValueOnce({
|
||||
embeddings: [{ values: dummyEmbedding }],
|
||||
})
|
||||
|
||||
const result = await service.generateEmbedding('beautiful sunset')
|
||||
|
||||
expect(result.embedding).toEqual(dummyEmbedding)
|
||||
expect(result.usage?.model).toBe('gemini-embedding-2')
|
||||
expect(result.cached).toBeUndefined()
|
||||
expect(mockEmbedContent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
model: 'gemini-embedding-2',
|
||||
contents: [
|
||||
expect.objectContaining({
|
||||
parts: [{ text: 'beautiful sunset' }],
|
||||
}),
|
||||
],
|
||||
config: {
|
||||
outputDimensionality: 1536,
|
||||
},
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('returns cached result on repeated call without re-invoking the provider', async () => {
|
||||
const dummyEmbedding = Array(1536).fill(0.05)
|
||||
mockEmbedContent.mockResolvedValueOnce({
|
||||
embeddings: [{ values: dummyEmbedding }],
|
||||
})
|
||||
|
||||
const first = await service.generateEmbedding('ocean breeze')
|
||||
expect(first.embedding).toEqual(dummyEmbedding)
|
||||
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
|
||||
|
||||
// Second call with same text and extra whitespace
|
||||
const second = await service.generateEmbedding(' ocean breeze ')
|
||||
expect(second.embedding).toEqual(dummyEmbedding)
|
||||
expect(second.cached).toBe(true)
|
||||
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('preserves case distinctions in cache keys', async () => {
|
||||
const dummyApple = Array(1536).fill(0.1)
|
||||
const dummyLowerApple = Array(1536).fill(0.2)
|
||||
mockEmbedContent
|
||||
.mockResolvedValueOnce({
|
||||
embeddings: [{ values: dummyApple }],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
embeddings: [{ values: dummyLowerApple }],
|
||||
})
|
||||
|
||||
const upper = await service.generateEmbedding('Apple')
|
||||
expect(upper.embedding).toEqual(dummyApple)
|
||||
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
|
||||
|
||||
const lower = await service.generateEmbedding('apple')
|
||||
expect(lower.embedding).toEqual(dummyLowerApple)
|
||||
expect(lower.cached).toBeUndefined()
|
||||
expect(mockEmbedContent).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('does not log raw query text on cache hits', async () => {
|
||||
const dummyEmbedding = Array(1536).fill(0.05)
|
||||
mockEmbedContent.mockResolvedValue({
|
||||
embeddings: [{ values: dummyEmbedding }],
|
||||
})
|
||||
const debugSpy = vi.spyOn(logger, 'debug')
|
||||
|
||||
await service.generateEmbedding('confidential search query')
|
||||
await service.generateEmbedding('confidential search query')
|
||||
|
||||
expect(debugSpy).toHaveBeenCalledWith(
|
||||
expect.not.objectContaining({
|
||||
key: expect.anything(),
|
||||
text: expect.anything(),
|
||||
}),
|
||||
'Cache hit for text embedding',
|
||||
)
|
||||
})
|
||||
|
||||
it('clears cache when clearCache is called', async () => {
|
||||
const dummyEmbedding = Array(1536).fill(0.05)
|
||||
mockEmbedContent.mockResolvedValue({
|
||||
embeddings: [{ values: dummyEmbedding }],
|
||||
})
|
||||
|
||||
await service.generateEmbedding('query')
|
||||
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
|
||||
|
||||
service.clearCache()
|
||||
|
||||
await service.generateEmbedding('query')
|
||||
expect(mockEmbedContent).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('throws if GEMINI_API_KEY is not configured', async () => {
|
||||
delete process.env.GEMINI_API_KEY
|
||||
await expect(service.generateEmbedding('test')).rejects.toThrow(
|
||||
'GEMINI_API_KEY environment variable is not configured',
|
||||
)
|
||||
})
|
||||
|
||||
it('throws if Gemini returns empty embedding values', async () => {
|
||||
mockEmbedContent.mockResolvedValueOnce({
|
||||
embeddings: [{ values: null }],
|
||||
})
|
||||
await expect(service.generateEmbedding('test')).rejects.toThrow(
|
||||
'No embedding values returned from gemini-embedding-2',
|
||||
)
|
||||
})
|
||||
|
||||
it('generates Gemma embedding (768d) when provider is gemma-embedding-2-openai', async () => {
|
||||
const dummyEmbedding = Array(768).fill(0.12)
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: dummyEmbedding }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 12 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const result = await service.generateEmbedding('drone footage', {
|
||||
provider: 'gemma-embedding-2-openai',
|
||||
})
|
||||
|
||||
expect(result.embedding).toEqual(dummyEmbedding)
|
||||
expect(result.usage?.model).toBe('embeddinggemma-2')
|
||||
expect(result.usage?.inputTokens).toBe(12)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input: 'task: search query | text: drone footage',
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('respects GEMMA_EMBEDDING_OPENAI_BASE_URL without /v1 suffix', async () => {
|
||||
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379'
|
||||
const dummyEmbedding = Array(768).fill(0.12)
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ embedding: dummyEmbedding }],
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
await service.generateEmbedding('test query', {
|
||||
provider: 'gemma-embedding-2-openai',
|
||||
})
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://custom-host:9379/v1/embeddings',
|
||||
expect.anything(),
|
||||
)
|
||||
})
|
||||
|
||||
it('throws descriptive error when Gemma server returns non-ok status', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 503,
|
||||
text: async () => 'Service Unavailable',
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
await expect(
|
||||
service.generateEmbedding('test query', {
|
||||
provider: 'gemma-embedding-2-openai',
|
||||
}),
|
||||
).rejects.toThrow('Embedding server returned status 503: Service Unavailable')
|
||||
})
|
||||
|
||||
it('throws error when abort signal is triggered', async () => {
|
||||
const controller = new AbortController()
|
||||
controller.abort(new Error('Operation cancelled by user'))
|
||||
|
||||
await expect(
|
||||
service.generateEmbedding('test query', {
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow('Operation cancelled by user')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,196 @@
|
||||
import { GoogleGenAI } from '@google/genai'
|
||||
import { LruTtlCache } from '@shumai/core/src/cache/lru-ttl-cache'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
export interface TextEmbeddingUsage {
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
}
|
||||
|
||||
export interface TextEmbeddingResult {
|
||||
embedding: number[]
|
||||
usage?: TextEmbeddingUsage
|
||||
cached?: boolean
|
||||
}
|
||||
|
||||
export interface GenerateTextEmbeddingOptions {
|
||||
provider?: string
|
||||
model?: string
|
||||
signal?: AbortSignal
|
||||
teamId?: string
|
||||
skipCache?: boolean
|
||||
}
|
||||
|
||||
export class TextEmbeddingService {
|
||||
private cache: LruTtlCache<string, number[]>
|
||||
|
||||
constructor(
|
||||
cacheCapacity: number = 500,
|
||||
private defaultTtlMs: number = 3600 * 1000,
|
||||
) {
|
||||
this.cache = new LruTtlCache<string, number[]>(cacheCapacity, 50)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a text embedding vector directly using the configured provider.
|
||||
* If a cached embedding exists for the text, provider, and model, it is returned immediately.
|
||||
*/
|
||||
async generateEmbedding(
|
||||
text: string,
|
||||
options?: GenerateTextEmbeddingOptions,
|
||||
): Promise<TextEmbeddingResult> {
|
||||
const trimmedText = text.trim()
|
||||
if (!trimmedText) {
|
||||
throw new Error('Text to embed must not be empty')
|
||||
}
|
||||
|
||||
const provider = options?.provider || process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
const model =
|
||||
options?.model ||
|
||||
(provider === 'gemma-embedding-2-openai' ? 'embeddinggemma-2' : 'gemini-embedding-2')
|
||||
|
||||
const cacheKey = `${provider}:${model}:${trimmedText}`
|
||||
if (!options?.skipCache) {
|
||||
const cachedVector = this.cache.get(cacheKey)
|
||||
if (cachedVector) {
|
||||
logger.debug({ provider, model }, 'Cache hit for text embedding')
|
||||
return {
|
||||
embedding: cachedVector,
|
||||
usage: { model, inputTokens: 0, outputTokens: 0 },
|
||||
cached: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const signal = options?.signal ?? AbortSignal.timeout(15000)
|
||||
|
||||
let result: TextEmbeddingResult
|
||||
if (provider === 'gemma-embedding-2-openai') {
|
||||
result = await this.callGemmaEmbeddingServer(trimmedText, model, signal)
|
||||
} else {
|
||||
result = await this.callGeminiEmbedding(trimmedText, model, signal)
|
||||
}
|
||||
|
||||
if (!options?.skipCache) {
|
||||
this.cache.set(cacheKey, result.embedding, this.defaultTtlMs)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private async callGeminiEmbedding(
|
||||
text: string,
|
||||
model: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<TextEmbeddingResult> {
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw new Error('GEMINI_API_KEY environment variable is not configured')
|
||||
}
|
||||
|
||||
if (signal.aborted) {
|
||||
throw signal.reason ?? new Error('Embedding generation aborted or timed out')
|
||||
}
|
||||
|
||||
const resolvedApiKey = process.env[apiKey] || apiKey
|
||||
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
|
||||
|
||||
const embedPromise = ai.models.embedContent({
|
||||
model,
|
||||
contents: [
|
||||
{
|
||||
parts: [{ text }],
|
||||
role: 'user',
|
||||
},
|
||||
],
|
||||
config: {
|
||||
outputDimensionality: 1536,
|
||||
},
|
||||
})
|
||||
|
||||
const abortPromise = new Promise<never>((_, reject) => {
|
||||
if (signal.aborted) {
|
||||
reject(signal.reason ?? new Error('Embedding generation aborted or timed out'))
|
||||
return
|
||||
}
|
||||
signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
reject(signal.reason ?? new Error('Embedding generation aborted or timed out'))
|
||||
},
|
||||
{ once: true },
|
||||
)
|
||||
})
|
||||
|
||||
const response = await Promise.race([embedPromise, abortPromise])
|
||||
const values = response.embeddings?.[0]?.values
|
||||
if (!values || !Array.isArray(values)) {
|
||||
throw new Error(`No embedding values returned from ${model}`)
|
||||
}
|
||||
|
||||
return {
|
||||
embedding: values,
|
||||
usage: {
|
||||
model,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
private async callGemmaEmbeddingServer(
|
||||
text: string,
|
||||
model: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<TextEmbeddingResult> {
|
||||
const rawBaseUrl = process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL || 'http://localhost:9379/v1'
|
||||
const normalized = rawBaseUrl.replace(/\/+$/, '')
|
||||
const endpoint = normalized.endsWith('/embeddings')
|
||||
? normalized
|
||||
: normalized.endsWith('/v1')
|
||||
? `${normalized}/embeddings`
|
||||
: `${normalized}/v1/embeddings`
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
input: `task: search query | text: ${text}`,
|
||||
}),
|
||||
signal,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
throw new Error(`Embedding server returned status ${response.status}: ${errorText}`)
|
||||
}
|
||||
|
||||
const json = (await response.json()) as {
|
||||
model?: string
|
||||
data?: Array<{ embedding?: number[] }>
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage?: { prompt_tokens?: number }
|
||||
}
|
||||
|
||||
const embedding = json.data?.[0]?.embedding
|
||||
if (!embedding || !Array.isArray(embedding)) {
|
||||
throw new Error('Invalid embedding returned from embedding server')
|
||||
}
|
||||
|
||||
return {
|
||||
embedding,
|
||||
usage: {
|
||||
model: json.model || model,
|
||||
inputTokens: json.usage?.prompt_tokens ?? 0,
|
||||
outputTokens: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
clearCache(): void {
|
||||
this.cache.clear()
|
||||
}
|
||||
}
|
||||
|
||||
export const textEmbeddingService = new TextEmbeddingService()
|
||||
@@ -375,6 +375,18 @@ describe('TeamService', () => {
|
||||
hlsEnabled: false,
|
||||
hlsResolutions: ['480p', '720p', '1080p'],
|
||||
},
|
||||
turntable: {
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
},
|
||||
gotenberg: {
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
},
|
||||
appearance: { hideAgent: false },
|
||||
})
|
||||
|
||||
@@ -392,6 +404,18 @@ describe('TeamService', () => {
|
||||
hlsEnabled: false,
|
||||
hlsResolutions: ['480p', '720p', '1080p'],
|
||||
},
|
||||
turntable: {
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
},
|
||||
gotenberg: {
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
},
|
||||
appearance: { hideAgent: false },
|
||||
})
|
||||
|
||||
@@ -418,6 +442,18 @@ describe('TeamService', () => {
|
||||
hlsEnabled: true,
|
||||
hlsResolutions: ['720p', '1080p', '2160p'],
|
||||
},
|
||||
turntable: {
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
},
|
||||
gotenberg: {
|
||||
hasPassword: false,
|
||||
isEnvConfigured: false,
|
||||
url: undefined,
|
||||
username: undefined,
|
||||
},
|
||||
appearance: { hideAgent: true },
|
||||
semanticSearchEnabled: false,
|
||||
})
|
||||
@@ -569,4 +605,32 @@ describe('TeamService', () => {
|
||||
expect(settings.appearance?.hideAgent).toBe(true)
|
||||
expect((settings as Record<string, unknown>).emailNotification).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should report semanticSearchEnabled false when ENABLE_EMBEDDING_AGENT is false even if agent is enabled in db', async () => {
|
||||
const owner = await prisma.user.create({
|
||||
data: { name: 'Owner', email: 'owner-emb-false@example.com', password: 'pw' },
|
||||
})
|
||||
const team = await teamService.createTeam(owner, { name: 'Emb False Team' })
|
||||
|
||||
const bot = await prisma.user.create({
|
||||
data: { name: 'Bot', email: 'bot-emb-false@example.com', password: 'pw', type: 'agent' },
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: bot.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: '', model: '' },
|
||||
},
|
||||
})
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
try {
|
||||
const settings = await teamService.getSettings(team.id)
|
||||
expect(settings.semanticSearchEnabled).toBe(false)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -6,7 +6,10 @@ import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import { getAvatarUrl } from '@shumai/core/src/user/avatar'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { getAllowedAgentRoles } from '@shumai/core/src/agent/permissions'
|
||||
import { AgentService, agentService } from '@shumai/core/src/agent/agent'
|
||||
import { sandboxService } from '@shumai/core/src/sandbox/sandbox-service'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
import {
|
||||
ServiceCreateTeamRequest,
|
||||
ServiceGetUserTeamsRequest,
|
||||
@@ -44,6 +47,7 @@ export class TeamService {
|
||||
},
|
||||
})
|
||||
await providerService.initBuiltinProviders(team.id)
|
||||
await agentService.ensureEmbeddingAgentForTeam(team.id)
|
||||
}
|
||||
|
||||
return team
|
||||
@@ -84,10 +88,12 @@ export class TeamService {
|
||||
},
|
||||
})
|
||||
|
||||
// We use 'any' here because ProviderService expects the full PrismaClient,
|
||||
// We use 'any' here because ProviderService/AgentService expect the full PrismaClient,
|
||||
// but the TransactionClient is compatible with the methods we use.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new ProviderService(tx as any).initBuiltinProviders(team.id)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new AgentService(tx as any).ensureEmbeddingAgentForTeam(team.id)
|
||||
|
||||
return this.toTeamInfo(updatedTeam)
|
||||
})
|
||||
@@ -305,9 +311,12 @@ export class TeamService {
|
||||
delete rawSettings.emailNotification
|
||||
const settings = rawSettings as TeamSettingsResponse
|
||||
|
||||
const embeddingAgent = await prisma.agent.findFirst({
|
||||
where: { teamId, type: 'embedding', enabled: true },
|
||||
})
|
||||
const embeddingAgent =
|
||||
process.env.ENABLE_EMBEDDING_AGENT === 'false'
|
||||
? null
|
||||
: await prisma.agent.findFirst({
|
||||
where: { teamId, type: 'embedding', enabled: true },
|
||||
})
|
||||
settings.semanticSearchEnabled = !!embeddingAgent
|
||||
|
||||
if (!settings.transcode) {
|
||||
@@ -343,6 +352,9 @@ export class TeamService {
|
||||
settings.appearance.hideAgent = false
|
||||
}
|
||||
|
||||
settings.turntable = await turntableService.getSettings(teamId)
|
||||
settings.gotenberg = await gotenbergService.getSettings(teamId)
|
||||
|
||||
return settings
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,11 @@ import * as fs from 'fs'
|
||||
import {
|
||||
extractEmbeddedJpeg,
|
||||
validateExtractedJpeg,
|
||||
validateExtractedImage,
|
||||
decodeRawWithDcraw,
|
||||
extractAndValidateRawPreview,
|
||||
EXIF_ORIENTATION_TO_ROTATION,
|
||||
withTimeout,
|
||||
} from './raw-extract'
|
||||
|
||||
// Mock exiftool-vendored
|
||||
@@ -26,13 +29,20 @@ vi.mock('sharp', () => {
|
||||
return { default: mockSharp }
|
||||
})
|
||||
|
||||
// Mock child_process
|
||||
vi.mock('child_process', () => ({
|
||||
execFile: vi.fn(),
|
||||
}))
|
||||
|
||||
import { exiftool } from 'exiftool-vendored'
|
||||
import sharp from 'sharp'
|
||||
import { execFile } from 'child_process'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const mockMetadata = (sharp as any)._mockMetadata as ReturnType<typeof vi.fn>
|
||||
const mockExtractBinaryTag = vi.mocked(exiftool.extractBinaryTag)
|
||||
const mockRead = vi.mocked(exiftool.read)
|
||||
const mockExecFile = vi.mocked(execFile)
|
||||
|
||||
describe('extractEmbeddedJpeg', () => {
|
||||
beforeEach(() => {
|
||||
@@ -162,6 +172,29 @@ describe('extractEmbeddedJpeg', () => {
|
||||
expect(result!.orientation).toBeUndefined()
|
||||
result!.cleanup()
|
||||
})
|
||||
|
||||
it('skips remaining tags and aborts early when extraction times out', async () => {
|
||||
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
try {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = '20'
|
||||
// Hang on first tag
|
||||
mockExtractBinaryTag.mockImplementationOnce(
|
||||
() => new Promise((resolve) => setTimeout(resolve, 300)),
|
||||
)
|
||||
|
||||
const result = await extractEmbeddedJpeg('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
// Crucial: should break immediately after the first timeout instead of trying remaining 2 tags
|
||||
expect(mockExtractBinaryTag).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
if (origEnv !== undefined) {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
|
||||
} else {
|
||||
delete process.env.EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateExtractedJpeg', () => {
|
||||
@@ -202,16 +235,265 @@ describe('validateExtractedJpeg', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateExtractedImage', () => {
|
||||
it('is an alias for validateExtractedJpeg', () => {
|
||||
expect(validateExtractedImage).toBe(validateExtractedJpeg)
|
||||
})
|
||||
})
|
||||
|
||||
describe('decodeRawWithDcraw', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('decodes RAW using camera white balance, sRGB, and TIFF output without -h when longest dimension <= 8192', async () => {
|
||||
mockRead.mockResolvedValueOnce({
|
||||
ImageWidth: 6000,
|
||||
ImageHeight: 4000,
|
||||
Orientation: 1,
|
||||
} as never)
|
||||
|
||||
let executedArgs: string[] = []
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
executedArgs = args as string[]
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(fs.existsSync(result!.previewPath)).toBe(true)
|
||||
expect(result!.orientation).toBeUndefined()
|
||||
expect(result!.rawWidth).toBe(6000)
|
||||
expect(result!.rawHeight).toBe(4000)
|
||||
|
||||
expect(executedArgs).toContain('-w')
|
||||
expect(executedArgs).toContain('-o')
|
||||
expect(executedArgs).toContain('1')
|
||||
expect(executedArgs).toContain('-T')
|
||||
expect(executedArgs).toContain('-Z')
|
||||
expect(executedArgs).not.toContain('-h')
|
||||
expect(executedArgs[executedArgs.length - 1]).toBe('/path/to/photo.cr2')
|
||||
|
||||
result!.cleanup()
|
||||
expect(fs.existsSync(result!.previewPath)).toBe(false)
|
||||
})
|
||||
|
||||
it('enables half-size (-h) when longest dimension > 8192px and computes upright raw dimensions', async () => {
|
||||
mockRead.mockResolvedValueOnce({
|
||||
ImageWidth: 9504,
|
||||
ImageHeight: 6336,
|
||||
Orientation: 6,
|
||||
} as never)
|
||||
|
||||
let executedArgs: string[] = []
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
executedArgs = args as string[]
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.arw')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(executedArgs).toContain('-h')
|
||||
expect(result!.orientation).toBeUndefined()
|
||||
// For orientation 6 (90 CW), upright raw dimensions are swapped
|
||||
expect(result!.rawWidth).toBe(6336)
|
||||
expect(result!.rawHeight).toBe(9504)
|
||||
|
||||
result!.cleanup()
|
||||
})
|
||||
|
||||
it('omits -h when longest dimension is exactly 8192px', async () => {
|
||||
mockRead.mockResolvedValueOnce({
|
||||
ImageWidth: 8192,
|
||||
ImageHeight: 5464,
|
||||
Orientation: 1,
|
||||
} as never)
|
||||
|
||||
let executedArgs: string[] = []
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
executedArgs = args as string[]
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.nef')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(executedArgs).not.toContain('-h')
|
||||
result!.cleanup()
|
||||
})
|
||||
|
||||
it('defaults to full size without -h when EXIF metadata cannot be read', async () => {
|
||||
mockRead.mockRejectedValueOnce(new Error('Exif read failure'))
|
||||
|
||||
let executedArgs: string[] = []
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
executedArgs = args as string[]
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.dng')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(executedArgs).not.toContain('-h')
|
||||
result!.cleanup()
|
||||
})
|
||||
|
||||
it('defaults to full size without -h when EXIF metadata read times out', async () => {
|
||||
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
try {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = '20'
|
||||
mockRead.mockImplementationOnce(() => new Promise((resolve) => setTimeout(resolve, 300)))
|
||||
|
||||
let executedArgs: string[] = []
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
executedArgs = args as string[]
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.dng')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(executedArgs).not.toContain('-h')
|
||||
result!.cleanup()
|
||||
} finally {
|
||||
if (origEnv !== undefined) {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
|
||||
} else {
|
||||
delete process.env.EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('returns null and cleans up when dcraw_emu exits with non-zero code', async () => {
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
const err = new Error('Command failed: dcraw_emu')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(err as any).code = 1
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(err)
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null and cleans up when dcraw_emu times out', async () => {
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
const err = new Error('Process timed out')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(err as any).code = 'ETIMEDOUT'
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(err)
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null and cleans up when dcraw_emu is not found (ENOENT)', async () => {
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
const err = new Error('spawn dcraw_emu ENOENT')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(err as any).code = 'ENOENT'
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(err)
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null and cleans up when output TIFF is missing or empty', async () => {
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractAndValidateRawPreview', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('returns previewPath, dimensions, orientation, and cleanup for valid RAW', async () => {
|
||||
it('returns previewPath, dimensions, orientation, and cleanup for valid RAW using embedded JPEG first', async () => {
|
||||
mockExtractBinaryTag.mockImplementationOnce(async (_tag, _src, dest) => {
|
||||
fs.writeFileSync(dest, 'jpeg-data')
|
||||
})
|
||||
mockRead.mockResolvedValueOnce({ Orientation: 6 } as never)
|
||||
mockRead.mockResolvedValueOnce({ Orientation: 6, ImageWidth: 4000, ImageHeight: 3000 } as never)
|
||||
mockMetadata.mockResolvedValueOnce({ width: 4000, height: 3000 })
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
@@ -221,35 +503,207 @@ describe('extractAndValidateRawPreview', () => {
|
||||
expect(result!.width).toBe(4000)
|
||||
expect(result!.height).toBe(3000)
|
||||
expect(result!.orientation).toBe(6)
|
||||
expect(mockExecFile).not.toHaveBeenCalled()
|
||||
|
||||
result!.cleanup()
|
||||
expect(fs.existsSync(result!.previewPath)).toBe(false)
|
||||
})
|
||||
|
||||
it('returns null when extraction fails', async () => {
|
||||
it('falls back to dcraw_emu when embedded preview extraction fails', async () => {
|
||||
mockExtractBinaryTag
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
|
||||
mockRead.mockResolvedValueOnce({
|
||||
ImageWidth: 6000,
|
||||
ImageHeight: 4000,
|
||||
Orientation: 1,
|
||||
} as never)
|
||||
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(mockExecFile).toHaveBeenCalled()
|
||||
expect(result!.width).toBe(6000)
|
||||
expect(result!.height).toBe(4000)
|
||||
expect(result!.rawWidth).toBe(6000)
|
||||
expect(result!.rawHeight).toBe(4000)
|
||||
expect(result!.orientation).toBeUndefined()
|
||||
|
||||
result!.cleanup()
|
||||
expect(fs.existsSync(result!.previewPath)).toBe(false)
|
||||
})
|
||||
|
||||
it('falls back to dcraw_emu when extracted embedded JPEG is corrupt', async () => {
|
||||
let createdJpegPath = ''
|
||||
mockExtractBinaryTag.mockImplementationOnce(async (_tag, _src, dest) => {
|
||||
createdJpegPath = dest
|
||||
fs.writeFileSync(dest, 'corrupt-jpeg')
|
||||
})
|
||||
mockRead.mockResolvedValueOnce({ Orientation: 1 } as never)
|
||||
mockMetadata.mockRejectedValueOnce(new Error('Not a valid image'))
|
||||
|
||||
// Fallback dcraw_emu
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(fs.existsSync(createdJpegPath)).toBe(false) // Embedded jpeg was cleaned up
|
||||
expect(fs.existsSync(result!.previewPath)).toBe(true)
|
||||
|
||||
result!.cleanup()
|
||||
expect(fs.existsSync(result!.previewPath)).toBe(false)
|
||||
})
|
||||
|
||||
it('returns null when both embedded extraction and dcraw_emu fail', async () => {
|
||||
mockExtractBinaryTag
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
|
||||
mockRead.mockRejectedValueOnce(new Error('fail'))
|
||||
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(new Error('dcraw failed'))
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null and cleans up when validation fails', async () => {
|
||||
let createdPath = ''
|
||||
mockExtractBinaryTag.mockImplementationOnce(async (_tag, _src, dest) => {
|
||||
createdPath = dest
|
||||
fs.writeFileSync(dest, 'bad-jpeg')
|
||||
it('returns null and cleans up when decoded TIFF is corrupt', async () => {
|
||||
mockExtractBinaryTag
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
.mockRejectedValueOnce(new Error('fail'))
|
||||
|
||||
let createdTiff = ''
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
createdTiff = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(createdTiff, 'corrupt-tiff')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
mockRead.mockResolvedValueOnce({ Orientation: 1 } as never)
|
||||
mockMetadata.mockRejectedValueOnce(new Error('Not a valid image'))
|
||||
mockMetadata.mockRejectedValueOnce(new Error('Invalid TIFF'))
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
expect(fs.existsSync(createdPath)).toBe(false)
|
||||
expect(fs.existsSync(createdTiff)).toBe(false)
|
||||
})
|
||||
|
||||
it('falls back to dcraw_emu when embedded preview extraction times out', async () => {
|
||||
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
try {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = '20'
|
||||
// Hang on extractBinaryTag
|
||||
mockExtractBinaryTag.mockImplementationOnce(
|
||||
() => new Promise((resolve) => setTimeout(resolve, 300)),
|
||||
)
|
||||
|
||||
mockRead.mockResolvedValueOnce({
|
||||
ImageWidth: 6000,
|
||||
ImageHeight: 4000,
|
||||
Orientation: 1,
|
||||
} as never)
|
||||
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(mockExecFile).toHaveBeenCalled()
|
||||
expect(result!.width).toBe(6000)
|
||||
expect(result!.height).toBe(4000)
|
||||
|
||||
result!.cleanup()
|
||||
} finally {
|
||||
if (origEnv !== undefined) {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
|
||||
} else {
|
||||
delete process.env.EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('withTimeout', () => {
|
||||
it('resolves when promise settles before timeout', async () => {
|
||||
const result = await withTimeout(Promise.resolve('ok'), 1000)
|
||||
expect(result).toBe('ok')
|
||||
})
|
||||
|
||||
it('rejects with timeout error when promise exceeds timeout', async () => {
|
||||
const hanging = new Promise((resolve) => setTimeout(resolve, 500))
|
||||
await expect(withTimeout(hanging, 20, 'TestTask')).rejects.toThrow(
|
||||
'TestTask timed out after 20ms',
|
||||
)
|
||||
})
|
||||
|
||||
it('propagates underlying promise rejection before timeout', async () => {
|
||||
const failing = Promise.reject(new Error('fail fast'))
|
||||
await expect(withTimeout(failing, 1000, 'TestTask')).rejects.toThrow('fail fast')
|
||||
})
|
||||
|
||||
it('prevents unhandled rejection if promise rejects after timeout', async () => {
|
||||
const delayedReject = new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('late error')), 40),
|
||||
)
|
||||
await expect(withTimeout(delayedReject, 10, 'TestTask')).rejects.toThrow(
|
||||
'TestTask timed out after 10ms',
|
||||
)
|
||||
await new Promise((resolve) => setTimeout(resolve, 60))
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,15 +1,42 @@
|
||||
import { execFile } from 'child_process'
|
||||
import { exiftool } from 'exiftool-vendored'
|
||||
import sharp from 'sharp'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import sharp from 'sharp'
|
||||
import { promisify } from 'util'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
export interface RawExtractResult {
|
||||
previewPath: string
|
||||
cleanup: () => void
|
||||
/** EXIF orientation from the RAW container (1–8), or undefined if absent. */
|
||||
/** EXIF orientation from the RAW container (1–8), or undefined if absent or already applied. */
|
||||
orientation?: number
|
||||
/** Original unoriented or oriented image dimensions from EXIF if available */
|
||||
rawWidth?: number
|
||||
rawHeight?: number
|
||||
}
|
||||
|
||||
export interface RawPreviewResult {
|
||||
previewPath: string
|
||||
cleanup: () => void
|
||||
width: number
|
||||
height: number
|
||||
orientation?: number
|
||||
/**
|
||||
* True when the file is already upright (dcraw_emu output), so downstream sharp must not
|
||||
* auto-orient it even though `orientation` is undefined.
|
||||
*/
|
||||
orientationApplied?: boolean
|
||||
rawWidth?: number
|
||||
rawHeight?: number
|
||||
}
|
||||
|
||||
export interface DecodeRawOptions {
|
||||
timeoutMs?: number
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -30,6 +57,41 @@ export const EXIF_ORIENTATION_TO_ROTATION: Record<
|
||||
8: { angle: 270 }, // Rotate 270° CW
|
||||
}
|
||||
|
||||
const DEFAULT_EXIFTOOL_TIMEOUT_MS = 15_000
|
||||
|
||||
function getExifToolTimeoutMs(): number {
|
||||
const envVal = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
if (envVal) {
|
||||
const parsed = parseInt(envVal, 10)
|
||||
if (!isNaN(parsed) && parsed > 0) {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return DEFAULT_EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a promise with a timeout. If the timeout triggers first,
|
||||
* the returned promise rejects with a timeout error and attaches a no-op handler
|
||||
* to the underlying promise to prevent unhandled rejections.
|
||||
*/
|
||||
export function withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
operationName = 'Operation',
|
||||
): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(new Error(`${operationName} timed out after ${timeoutMs}ms`))
|
||||
}, timeoutMs)
|
||||
})
|
||||
promise.catch(() => {})
|
||||
return Promise.race([promise, timeoutPromise]).finally(() => {
|
||||
if (timer) clearTimeout(timer)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to extract an embedded JPEG preview from a camera RAW file to a temporary file.
|
||||
*
|
||||
@@ -61,32 +123,58 @@ export async function extractEmbeddedJpeg(
|
||||
}
|
||||
}
|
||||
|
||||
const exiftoolTimeoutMs = getExifToolTimeoutMs()
|
||||
|
||||
for (const tag of tags) {
|
||||
const candidatePath = path.join(workDir, `preview-${tag}-${Date.now()}.jpg`)
|
||||
try {
|
||||
await exiftool.extractBinaryTag(tag, rawFilePath, candidatePath)
|
||||
await withTimeout(
|
||||
exiftool.extractBinaryTag(tag, rawFilePath, candidatePath),
|
||||
exiftoolTimeoutMs,
|
||||
`exiftool.extractBinaryTag(${tag})`,
|
||||
)
|
||||
if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).size > 0) {
|
||||
logger.debug({ tag, rawFilePath, candidatePath }, 'Extracted embedded JPEG from RAW')
|
||||
|
||||
// Read orientation from the RAW container's EXIF metadata.
|
||||
// The extracted JPEG binary stream often does NOT carry its own
|
||||
// orientation tag, so we must read it from the parent RAW file
|
||||
// to avoid incorrectly-rotated previews.
|
||||
// Read orientation and container dimensions from the RAW container's EXIF metadata.
|
||||
let orientation: number | undefined
|
||||
let rawWidth: number | undefined
|
||||
let rawHeight: number | undefined
|
||||
try {
|
||||
const exif = await exiftool.read(rawFilePath)
|
||||
const exif = await withTimeout(
|
||||
exiftool.read(rawFilePath),
|
||||
exiftoolTimeoutMs,
|
||||
'exiftool.read',
|
||||
)
|
||||
const rawOrientation = exif.Orientation
|
||||
if (typeof rawOrientation === 'number' && rawOrientation >= 1 && rawOrientation <= 8) {
|
||||
orientation = rawOrientation
|
||||
}
|
||||
const w =
|
||||
typeof exif.ImageWidth === 'number'
|
||||
? exif.ImageWidth
|
||||
: typeof exif.ExifImageWidth === 'number'
|
||||
? exif.ExifImageWidth
|
||||
: undefined
|
||||
const h =
|
||||
typeof exif.ImageHeight === 'number'
|
||||
? exif.ImageHeight
|
||||
: typeof exif.ExifImageHeight === 'number'
|
||||
? exif.ExifImageHeight
|
||||
: undefined
|
||||
if (w && h && w > 0 && h > 0) {
|
||||
rawWidth = w
|
||||
rawHeight = h
|
||||
}
|
||||
} catch {
|
||||
logger.debug({ rawFilePath }, 'Could not read EXIF orientation from RAW container')
|
||||
}
|
||||
|
||||
return { previewPath: candidatePath, cleanup, orientation }
|
||||
return { previewPath: candidatePath, cleanup, orientation, rawWidth, rawHeight }
|
||||
}
|
||||
} catch {
|
||||
logger.debug({ tag, rawFilePath }, `Could not extract ${tag} from RAW file`)
|
||||
} catch (err) {
|
||||
const isTimeout = err instanceof Error && err.message.includes('timed out')
|
||||
logger.debug({ tag, rawFilePath, err, isTimeout }, `Could not extract ${tag} from RAW file`)
|
||||
try {
|
||||
if (fs.existsSync(candidatePath)) {
|
||||
fs.rmSync(candidatePath, { force: true })
|
||||
@@ -94,6 +182,13 @@ export async function extractEmbeddedJpeg(
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (isTimeout) {
|
||||
logger.warn(
|
||||
{ rawFilePath, tag, timeoutMs: exiftoolTimeoutMs },
|
||||
'ExifTool extraction timed out; skipping remaining tags to avoid further delays',
|
||||
)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,10 +198,124 @@ export async function extractEmbeddedJpeg(
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that an extracted JPEG file is decodable.
|
||||
* Fallback RAW decoder using dcraw_emu (from LibRaw).
|
||||
*
|
||||
* Used when no usable embedded JPEG preview exists in the RAW file.
|
||||
*
|
||||
* Decoding options:
|
||||
* - Camera white balance and sRGB output (-w -o 1)
|
||||
* - Output temporary TIFF (-T -Z <tempPath>)
|
||||
* - Half-size processing (-h) enabled only when longest dimension > 8192px
|
||||
*
|
||||
* LibRaw auto-rotates the output image to upright orientation based on camera EXIF,
|
||||
* so the returned orientation is undefined to avoid double-rotation in downstream Sharp.
|
||||
*/
|
||||
export async function decodeRawWithDcraw(
|
||||
rawFilePath: string,
|
||||
targetDir?: string,
|
||||
options?: DecodeRawOptions,
|
||||
): Promise<RawExtractResult | null> {
|
||||
const ownTempDir = !targetDir
|
||||
const workDir = targetDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'raw-decode-'))
|
||||
const tempTiffPath = path.join(
|
||||
workDir,
|
||||
`decoded-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.tiff`,
|
||||
)
|
||||
|
||||
const cleanup = () => {
|
||||
try {
|
||||
if (ownTempDir && fs.existsSync(workDir)) {
|
||||
fs.rmSync(workDir, { recursive: true, force: true })
|
||||
} else if (fs.existsSync(tempTiffPath)) {
|
||||
fs.rmSync(tempTiffPath, { force: true })
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ workDir, tempTiffPath, err }, 'Failed to clean up dcraw temporary files')
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Probe RAW dimensions and orientation via ExifTool to check for 8192px threshold
|
||||
let useHalfSize = false
|
||||
let rawWidth: number | undefined
|
||||
let rawHeight: number | undefined
|
||||
try {
|
||||
const exiftoolTimeoutMs = getExifToolTimeoutMs()
|
||||
const exif = await withTimeout(exiftool.read(rawFilePath), exiftoolTimeoutMs, 'exiftool.read')
|
||||
const w =
|
||||
typeof exif.ImageWidth === 'number'
|
||||
? exif.ImageWidth
|
||||
: typeof exif.ExifImageWidth === 'number'
|
||||
? exif.ExifImageWidth
|
||||
: undefined
|
||||
const h =
|
||||
typeof exif.ImageHeight === 'number'
|
||||
? exif.ImageHeight
|
||||
: typeof exif.ExifImageHeight === 'number'
|
||||
? exif.ExifImageHeight
|
||||
: undefined
|
||||
|
||||
if (w && h && w > 0 && h > 0) {
|
||||
const longest = Math.max(w, h)
|
||||
if (longest > 8192) {
|
||||
useHalfSize = true
|
||||
}
|
||||
|
||||
// Compute upright raw dimensions matching dcraw_emu's auto-rotation
|
||||
const rawOrientation = exif.Orientation
|
||||
const isSwapped =
|
||||
typeof rawOrientation === 'number' && rawOrientation >= 5 && rawOrientation <= 8
|
||||
rawWidth = isSwapped ? h : w
|
||||
rawHeight = isSwapped ? w : h
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug({ rawFilePath, err }, 'Could not read EXIF metadata for RAW dimension check')
|
||||
}
|
||||
|
||||
// 2. Build dcraw_emu arguments
|
||||
const args = ['-w', '-o', '1']
|
||||
if (useHalfSize) {
|
||||
args.push('-h')
|
||||
}
|
||||
args.push('-T', '-Z', tempTiffPath, rawFilePath)
|
||||
|
||||
const dcrawBin = process.env.DCRAW_EMU_PATH || 'dcraw_emu'
|
||||
const envTimeout = process.env.DCRAW_EMU_TIMEOUT_MS
|
||||
? parseInt(process.env.DCRAW_EMU_TIMEOUT_MS, 10)
|
||||
: undefined
|
||||
const timeoutMs = options?.timeoutMs ?? (envTimeout && !isNaN(envTimeout) ? envTimeout : 60_000)
|
||||
|
||||
try {
|
||||
logger.debug({ dcrawBin, args, rawFilePath }, 'Executing dcraw_emu fallback')
|
||||
await execFileAsync(dcrawBin, args, {
|
||||
timeout: timeoutMs,
|
||||
signal: options?.signal,
|
||||
})
|
||||
|
||||
if (!fs.existsSync(tempTiffPath) || fs.statSync(tempTiffPath).size === 0) {
|
||||
logger.warn({ rawFilePath, tempTiffPath }, 'dcraw_emu did not produce a valid output TIFF')
|
||||
cleanup()
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
previewPath: tempTiffPath,
|
||||
cleanup,
|
||||
orientation: undefined, // dcraw_emu already auto-rotates upright based on camera EXIF
|
||||
rawWidth,
|
||||
rawHeight,
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn({ rawFilePath, err }, 'dcraw_emu failed to decode RAW file')
|
||||
cleanup()
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that an extracted or decoded image file (JPEG, TIFF, etc.) is decodable.
|
||||
* Returns dimensions if valid, null if corrupt/unusable.
|
||||
*/
|
||||
export async function validateExtractedJpeg(
|
||||
export async function validateExtractedImage(
|
||||
filePath: string,
|
||||
): Promise<{ width: number; height: number } | null> {
|
||||
try {
|
||||
@@ -120,34 +329,67 @@ export async function validateExtractedJpeg(
|
||||
}
|
||||
}
|
||||
|
||||
/** Backward compatibility alias */
|
||||
export const validateExtractedJpeg = validateExtractedImage
|
||||
|
||||
/**
|
||||
* Extract and validate an embedded JPEG from a RAW file to a temporary file.
|
||||
* Extract and validate a preview from a RAW file to a temporary file.
|
||||
*
|
||||
* 1. Attempts ExifTool embedded JPEG extraction first.
|
||||
* 2. If no embedded preview is available or extraction fails, falls back to dcraw_emu decoding.
|
||||
*
|
||||
* Returns a validated file path + dimensions + orientation + cleanup function, or null.
|
||||
*/
|
||||
export async function extractAndValidateRawPreview(
|
||||
rawFilePath: string,
|
||||
targetDir?: string,
|
||||
): Promise<{
|
||||
previewPath: string
|
||||
cleanup: () => void
|
||||
width: number
|
||||
height: number
|
||||
orientation?: number
|
||||
} | null> {
|
||||
): Promise<RawPreviewResult | null> {
|
||||
// 1. Try ExifTool embedded JPEG extraction first
|
||||
const extracted = await extractEmbeddedJpeg(rawFilePath, targetDir)
|
||||
if (!extracted) return null
|
||||
|
||||
const dims = await validateExtractedJpeg(extracted.previewPath)
|
||||
if (!dims) {
|
||||
logger.warn({ rawFilePath }, 'Extracted JPEG from RAW is not decodable, skipping preview')
|
||||
if (extracted) {
|
||||
const dims = await validateExtractedImage(extracted.previewPath)
|
||||
if (dims) {
|
||||
return {
|
||||
previewPath: extracted.previewPath,
|
||||
cleanup: extracted.cleanup,
|
||||
orientation: extracted.orientation,
|
||||
rawWidth: extracted.rawWidth,
|
||||
rawHeight: extracted.rawHeight,
|
||||
...dims,
|
||||
}
|
||||
}
|
||||
logger.warn(
|
||||
{ rawFilePath },
|
||||
'Extracted JPEG from RAW is not decodable, falling back to dcraw_emu',
|
||||
)
|
||||
extracted.cleanup()
|
||||
} else {
|
||||
logger.info(
|
||||
{ rawFilePath },
|
||||
'No usable embedded JPEG found in RAW file, falling back to dcraw_emu',
|
||||
)
|
||||
}
|
||||
|
||||
// 2. Fallback to dcraw_emu
|
||||
const decoded = await decodeRawWithDcraw(rawFilePath, targetDir)
|
||||
if (!decoded) {
|
||||
return null
|
||||
}
|
||||
|
||||
const dims = await validateExtractedImage(decoded.previewPath)
|
||||
if (!dims) {
|
||||
logger.warn({ rawFilePath }, 'Decoded TIFF from dcraw_emu is not decodable, skipping preview')
|
||||
decoded.cleanup()
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
previewPath: extracted.previewPath,
|
||||
cleanup: extracted.cleanup,
|
||||
orientation: extracted.orientation,
|
||||
previewPath: decoded.previewPath,
|
||||
cleanup: decoded.cleanup,
|
||||
orientation: decoded.orientation,
|
||||
orientationApplied: true,
|
||||
rawWidth: decoded.rawWidth,
|
||||
rawHeight: decoded.rawHeight,
|
||||
...dims,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import sharp from 'sharp'
|
||||
import { transcodeService } from './transcode'
|
||||
|
||||
// Only the storage layer is mocked: these tests run the real sharp pipeline end to end.
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
downloadToFile: vi.fn(),
|
||||
putObject: vi.fn(),
|
||||
resolveInput: vi.fn(),
|
||||
presign: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
// Only the RAW preview extraction is replaced, so the RAW branch of transcodeImage can be driven
|
||||
// with a prepared file (the dcraw_emu fallback result) without ExifTool or dcraw_emu installed.
|
||||
const extractRawMock = vi.hoisted(() => vi.fn())
|
||||
vi.mock('./raw-extract', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('./raw-extract')>()),
|
||||
extractAndValidateRawPreview: extractRawMock,
|
||||
}))
|
||||
|
||||
// Windows keeps cached files open, which would make the temp dir cleanup fail with EBUSY.
|
||||
sharp.cache(false)
|
||||
|
||||
type Corner = 'TL' | 'TR' | 'BL' | 'BR'
|
||||
|
||||
// Stored (sensor) pixels are landscape. Each quadrant has its own color so the direction of the
|
||||
// rotation or flip can be read back from the corners of the output.
|
||||
const STORED_WIDTH = 64
|
||||
const STORED_HEIGHT = 48
|
||||
const QUADRANT_COLORS: Record<Corner, [number, number, number]> = {
|
||||
TL: [220, 30, 30], // red
|
||||
TR: [30, 200, 30], // green
|
||||
BL: [30, 30, 220], // blue
|
||||
BR: [230, 230, 30], // yellow
|
||||
}
|
||||
|
||||
/**
|
||||
* For each EXIF orientation: the displayed size and which stored corner ends up at the displayed
|
||||
* top-left and bottom-right. Orientations 5 to 8 swap width and height.
|
||||
*/
|
||||
const ORIENTATIONS: Array<{
|
||||
orientation: number
|
||||
name: string
|
||||
swapped: boolean
|
||||
topLeftIs: Corner
|
||||
bottomRightIs: Corner
|
||||
}> = [
|
||||
{ orientation: 1, name: 'normal', swapped: false, topLeftIs: 'TL', bottomRightIs: 'BR' },
|
||||
{
|
||||
orientation: 2,
|
||||
name: 'mirror horizontal',
|
||||
swapped: false,
|
||||
topLeftIs: 'TR',
|
||||
bottomRightIs: 'BL',
|
||||
},
|
||||
{ orientation: 3, name: 'rotate 180', swapped: false, topLeftIs: 'BR', bottomRightIs: 'TL' },
|
||||
{ orientation: 4, name: 'mirror vertical', swapped: false, topLeftIs: 'BL', bottomRightIs: 'TR' },
|
||||
{ orientation: 5, name: 'transpose', swapped: true, topLeftIs: 'TL', bottomRightIs: 'BR' },
|
||||
{ orientation: 6, name: 'rotate 90 CW', swapped: true, topLeftIs: 'BL', bottomRightIs: 'TR' },
|
||||
{ orientation: 7, name: 'transverse', swapped: true, topLeftIs: 'BR', bottomRightIs: 'TL' },
|
||||
{ orientation: 8, name: 'rotate 270 CW', swapped: true, topLeftIs: 'TR', bottomRightIs: 'BL' },
|
||||
]
|
||||
|
||||
async function makeJpeg(orientation: number): Promise<Buffer> {
|
||||
const halfW = STORED_WIDTH / 2
|
||||
const halfH = STORED_HEIGHT / 2
|
||||
const block = (corner: Corner) => ({
|
||||
create: {
|
||||
width: halfW,
|
||||
height: halfH,
|
||||
channels: 3 as const,
|
||||
background: {
|
||||
r: QUADRANT_COLORS[corner][0],
|
||||
g: QUADRANT_COLORS[corner][1],
|
||||
b: QUADRANT_COLORS[corner][2],
|
||||
},
|
||||
},
|
||||
})
|
||||
const tile = async (corner: Corner) => sharp(block(corner)).png().toBuffer()
|
||||
return sharp({
|
||||
create: {
|
||||
width: STORED_WIDTH,
|
||||
height: STORED_HEIGHT,
|
||||
channels: 3,
|
||||
background: { r: 0, g: 0, b: 0 },
|
||||
},
|
||||
})
|
||||
.composite([
|
||||
{ input: await tile('TL'), left: 0, top: 0 },
|
||||
{ input: await tile('TR'), left: halfW, top: 0 },
|
||||
{ input: await tile('BL'), left: 0, top: halfH },
|
||||
{ input: await tile('BR'), left: halfW, top: halfH },
|
||||
])
|
||||
.jpeg({ quality: 100, chromaSubsampling: '4:4:4' })
|
||||
.withMetadata({ orientation })
|
||||
.toBuffer()
|
||||
}
|
||||
|
||||
/** Which quadrant color is nearest to the pixel a few pixels inside the given output corner. */
|
||||
async function cornerColor(file: string, where: 'TL' | 'BR'): Promise<Corner> {
|
||||
const { data, info } = await sharp(fs.readFileSync(file))
|
||||
.removeAlpha()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true })
|
||||
const inset = 3
|
||||
const x = where === 'TL' ? inset : info.width - 1 - inset
|
||||
const y = where === 'TL' ? inset : info.height - 1 - inset
|
||||
const offset = (y * info.width + x) * info.channels
|
||||
const pixel = [data[offset], data[offset + 1], data[offset + 2]]
|
||||
let best: Corner = 'TL'
|
||||
let bestDistance = Infinity
|
||||
for (const corner of Object.keys(QUADRANT_COLORS) as Corner[]) {
|
||||
const distance = QUADRANT_COLORS[corner].reduce(
|
||||
(sum, channel, i) => sum + (channel - pixel[i]) ** 2,
|
||||
0,
|
||||
)
|
||||
if (distance < bestDistance) {
|
||||
best = corner
|
||||
bestDistance = distance
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
describe('EXIF orientation (real sharp)', () => {
|
||||
let tempDir: string
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = transcodeService.createTempDir('orientation-test-')
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
transcodeService.removeDir(tempDir)
|
||||
})
|
||||
|
||||
const expectedSize = (swapped: boolean) =>
|
||||
swapped
|
||||
? { width: STORED_HEIGHT, height: STORED_WIDTH }
|
||||
: { width: STORED_WIDTH, height: STORED_HEIGHT }
|
||||
|
||||
it.each(ORIENTATIONS)(
|
||||
'getImageInfo reports the displayed size for orientation $orientation ($name)',
|
||||
async ({ orientation, swapped }) => {
|
||||
const inputPath = path.join(tempDir, `in-${orientation}.jpg`)
|
||||
fs.writeFileSync(inputPath, await makeJpeg(orientation))
|
||||
|
||||
const info = await transcodeService.getImageInfo(inputPath)
|
||||
const expected = expectedSize(swapped)
|
||||
|
||||
expect(info.originalWidth).toBe(expected.width)
|
||||
expect(info.originalHeight).toBe(expected.height)
|
||||
},
|
||||
)
|
||||
|
||||
it.each(ORIENTATIONS)(
|
||||
'transcodeImage outputs an upright image for orientation $orientation ($name)',
|
||||
async ({ orientation, swapped, topLeftIs, bottomRightIs }) => {
|
||||
const inputPath = path.join(tempDir, `in-${orientation}.jpg`)
|
||||
const outputPath = path.join(tempDir, `out-${orientation}.webp`)
|
||||
fs.writeFileSync(inputPath, await makeJpeg(orientation))
|
||||
|
||||
await transcodeService.transcodeImage(inputPath, outputPath, 1000, 100)
|
||||
|
||||
const meta = await sharp(fs.readFileSync(outputPath)).metadata()
|
||||
const expected = expectedSize(swapped)
|
||||
expect(meta.width).toBe(expected.width)
|
||||
expect(meta.height).toBe(expected.height)
|
||||
// Direction, not just size: the stored corner that lands top-left and bottom-right.
|
||||
expect(await cornerColor(outputPath, 'TL')).toBe(topLeftIs)
|
||||
expect(await cornerColor(outputPath, 'BR')).toBe(bottomRightIs)
|
||||
},
|
||||
)
|
||||
|
||||
it('transcodeImage sizes a rotated preview from the displayed dimensions', async () => {
|
||||
// Stored 64x48 with orientation 6 is displayed 48x64 (portrait): the short edge is the width.
|
||||
const inputPath = path.join(tempDir, 'preview-in.jpg')
|
||||
const outputPath = path.join(tempDir, 'preview-out.webp')
|
||||
fs.writeFileSync(inputPath, await makeJpeg(6))
|
||||
|
||||
await transcodeService.transcodeImage(inputPath, outputPath, 24, 80, { isPreview: true })
|
||||
|
||||
const meta = await sharp(fs.readFileSync(outputPath)).metadata()
|
||||
expect(meta.width).toBeLessThan(meta.height ?? 0)
|
||||
})
|
||||
|
||||
describe('RAW inputs', () => {
|
||||
const rawPreview = (previewPath: string, extra: Record<string, unknown>) => ({
|
||||
previewPath,
|
||||
cleanup: () => {},
|
||||
width: STORED_WIDTH,
|
||||
height: STORED_HEIGHT,
|
||||
...extra,
|
||||
})
|
||||
|
||||
it('never auto-orients dcraw_emu output that still carries an Orientation tag', async () => {
|
||||
// The dcraw fallback reports orientation undefined (already upright) but flags it as applied.
|
||||
const tiffLike = path.join(tempDir, 'dcraw.jpg')
|
||||
const outputPath = path.join(tempDir, 'dcraw-out.webp')
|
||||
fs.writeFileSync(tiffLike, await makeJpeg(6))
|
||||
extractRawMock.mockResolvedValueOnce(
|
||||
rawPreview(tiffLike, { orientation: undefined, orientationApplied: true }),
|
||||
)
|
||||
|
||||
await transcodeService.transcodeImage(path.join(tempDir, 'photo.cr2'), outputPath, 1000, 100)
|
||||
|
||||
const meta = await sharp(fs.readFileSync(outputPath)).metadata()
|
||||
// Stored landscape stays landscape and the stored corners stay put: no second rotation.
|
||||
expect(meta.width).toBe(STORED_WIDTH)
|
||||
expect(meta.height).toBe(STORED_HEIGHT)
|
||||
expect(await cornerColor(outputPath, 'TL')).toBe('TL')
|
||||
expect(await cornerColor(outputPath, 'BR')).toBe('BR')
|
||||
})
|
||||
|
||||
it('still auto-orients an embedded preview that has no container orientation', async () => {
|
||||
const embedded = path.join(tempDir, 'embedded.jpg')
|
||||
const outputPath = path.join(tempDir, 'embedded-out.webp')
|
||||
fs.writeFileSync(embedded, await makeJpeg(6))
|
||||
extractRawMock.mockResolvedValueOnce(rawPreview(embedded, { orientation: undefined }))
|
||||
|
||||
await transcodeService.transcodeImage(path.join(tempDir, 'photo.cr2'), outputPath, 1000, 100)
|
||||
|
||||
const meta = await sharp(fs.readFileSync(outputPath)).metadata()
|
||||
expect(meta.width).toBe(STORED_HEIGHT)
|
||||
expect(meta.height).toBe(STORED_WIDTH)
|
||||
})
|
||||
})
|
||||
|
||||
it('overlayAnnotationsOnBuffer keeps the displayed orientation', async () => {
|
||||
const annotations = [
|
||||
{
|
||||
type: 'box',
|
||||
color: '#ffffff',
|
||||
points: [
|
||||
[0.1, 0.1],
|
||||
[0.4, 0.4],
|
||||
],
|
||||
},
|
||||
] as unknown as PrismaJson.AnnotationList
|
||||
const annotated = await transcodeService.overlayAnnotationsOnBuffer(
|
||||
await makeJpeg(6),
|
||||
annotations,
|
||||
)
|
||||
|
||||
const meta = await sharp(annotated).metadata()
|
||||
expect(meta.width).toBe(STORED_HEIGHT)
|
||||
expect(meta.height).toBe(STORED_WIDTH)
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,7 +8,7 @@ import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import PDFDocument from 'pdfkit'
|
||||
import sharp from 'sharp'
|
||||
import sharp, { type Metadata as SharpMetadata } from 'sharp'
|
||||
import { ulid } from 'ulid'
|
||||
import { promisify } from 'util'
|
||||
import { mapConcurrent } from '../utils/async'
|
||||
@@ -194,6 +194,8 @@ export interface TranscodeVideoParams {
|
||||
sourceColorTransfer?: string
|
||||
sourceColorPrimaries?: string
|
||||
sourceColorSpace?: string
|
||||
/** Source rotation in degrees from MediaMetadata; probed when omitted. */
|
||||
sourceRotation?: number
|
||||
streamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
@@ -218,6 +220,8 @@ export interface TranscodeHlsRenditionParams {
|
||||
sourceColorPrimaries?: string
|
||||
sourceColorSpace?: string
|
||||
segmentDuration?: number
|
||||
/** Source rotation in degrees from MediaMetadata; probed when omitted. */
|
||||
sourceRotation?: number
|
||||
streamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
@@ -240,6 +244,10 @@ export const H264_ENCODER_CONFIGS: Record<string, EncoderConfig> = {
|
||||
name: 'h264_vaapi',
|
||||
presetArgs: ['-compression_level', '4'],
|
||||
},
|
||||
h264_rkmpp: {
|
||||
name: 'h264_rkmpp',
|
||||
presetArgs: ['-level', '51', '-rc_mode', 'AVBR'],
|
||||
},
|
||||
h264_amf: {
|
||||
name: 'h264_amf',
|
||||
presetArgs: ['-quality', 'balanced', '-rc', 'qvbr', '-qvbr_quality_level', '26'],
|
||||
@@ -254,6 +262,82 @@ export const H264_ENCODER_CONFIGS: Record<string, EncoderConfig> = {
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* How an encoder can also decode and scale on its own device, so frames stay in
|
||||
* GPU memory instead of being decoded and scaled on the CPU and uploaded.
|
||||
* Matches Immich's hardware acceleration pipelines for NVENC (CUDA), QSV, VAAPI,
|
||||
* and RKMPP.
|
||||
*/
|
||||
export interface HardwareDecodeConfig {
|
||||
/** Input options placed before `-i` (after any `-init_hw_device`). */
|
||||
inputArgs: string[]
|
||||
/** Dynamic input options generator taking device info if applicable. */
|
||||
getInputArgs?: (driDevice?: string) => string[]
|
||||
/** Output options placed after the filter graph. */
|
||||
outputArgs: string[]
|
||||
/** GPU filter fitting the frame within width x height with even dimensions. */
|
||||
scaleFilter: (width: number, height: number) => string
|
||||
}
|
||||
|
||||
export function resolveHwDecodeInputArgs(
|
||||
config: HardwareDecodeConfig,
|
||||
driDevice?: string,
|
||||
): string[] {
|
||||
return config.getInputArgs ? config.getInputArgs(driDevice) : config.inputArgs
|
||||
}
|
||||
|
||||
export const HW_DECODE_CONFIGS: Partial<Record<string, HardwareDecodeConfig>> = {
|
||||
h264_nvenc: {
|
||||
inputArgs: ['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda', '-threads', '1'],
|
||||
outputArgs: ['-noautoscale'],
|
||||
scaleFilter: (width, height) =>
|
||||
`scale_cuda=w=${width}:h=${height}:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12`,
|
||||
},
|
||||
h264_qsv: {
|
||||
inputArgs: [
|
||||
'-hwaccel',
|
||||
'qsv',
|
||||
'-hwaccel_output_format',
|
||||
'qsv',
|
||||
'-async_depth',
|
||||
'4',
|
||||
'-threads',
|
||||
'1',
|
||||
],
|
||||
getInputArgs: (driDevice?: string) => [
|
||||
'-hwaccel',
|
||||
'qsv',
|
||||
'-hwaccel_output_format',
|
||||
'qsv',
|
||||
'-async_depth',
|
||||
'4',
|
||||
'-threads',
|
||||
'1',
|
||||
...(driDevice ? ['-qsv_device', driDevice] : []),
|
||||
],
|
||||
outputArgs: ['-noautoscale'],
|
||||
scaleFilter: (width, height) =>
|
||||
`scale_qsv=w=${width}:h=${height}:async_depth=4:mode=hq:format=nv12`,
|
||||
},
|
||||
h264_vaapi: {
|
||||
inputArgs: ['-hwaccel', 'vaapi', '-hwaccel_device', 'accel', '-hwaccel_output_format', 'vaapi'],
|
||||
// If the decoder reinitializes mid-stream with a different coded size
|
||||
// (e.g. 1080 -> 1088), ffmpeg would otherwise auto-insert a software
|
||||
// scaler, which cannot take GPU frames, and fail.
|
||||
outputArgs: ['-noautoscale'],
|
||||
// Output range is left as-is (matching the software path): on i965,
|
||||
// out_range only retags the stream without converting levels.
|
||||
scaleFilter: (width, height) =>
|
||||
`scale_vaapi=w=${width}:h=${height}:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12:mode=hq`,
|
||||
},
|
||||
h264_rkmpp: {
|
||||
inputArgs: ['-hwaccel', 'rkmpp', '-hwaccel_output_format', 'drm_prime', '-afbc', 'rga'],
|
||||
outputArgs: ['-noautoscale'],
|
||||
scaleFilter: (width, height) =>
|
||||
`scale_rkrga=w=${width}:h=${height}:format=nv12:afbc=1:async_depth=4`,
|
||||
},
|
||||
}
|
||||
|
||||
export function getPlatformEncoderCandidates(
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): string[] {
|
||||
@@ -264,12 +348,13 @@ export function getPlatformEncoderCandidates(
|
||||
return ['h264_nvenc', 'h264_qsv', 'h264_amf']
|
||||
case 'linux':
|
||||
default:
|
||||
return ['h264_nvenc', 'h264_vaapi', 'h264_qsv', 'h264_amf']
|
||||
return ['h264_nvenc', 'h264_vaapi', 'h264_qsv', 'h264_rkmpp', 'h264_amf']
|
||||
}
|
||||
}
|
||||
|
||||
export function getVaapiDevice(driDir = '/dev/dri'): string | null {
|
||||
const envDevice = process.env.SHUMAI_VAAPI_DEVICE || process.env.VAAPI_DEVICE
|
||||
export function getDriDevice(driDir = '/dev/dri'): string | null {
|
||||
const envDevice =
|
||||
process.env.SHUMAI_HW_DEVICE || process.env.SHUMAI_VAAPI_DEVICE || process.env.VAAPI_DEVICE
|
||||
if (envDevice) {
|
||||
return envDevice
|
||||
}
|
||||
@@ -293,6 +378,69 @@ export function getVaapiDevice(driDir = '/dev/dri'): string | null {
|
||||
return null
|
||||
}
|
||||
|
||||
export function getVaapiDevice(driDir = '/dev/dri'): string | null {
|
||||
return getDriDevice(driDir)
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether hardware encoders with a {@link HW_DECODE_CONFIGS} entry also decode
|
||||
* and scale on the GPU. Enabled by default; set SHUMAI_HW_DECODE=false to disable.
|
||||
*/
|
||||
export function isHwDecodeEnabled(): boolean {
|
||||
const value = process.env.SHUMAI_HW_DECODE?.trim().toLowerCase()
|
||||
return value !== 'false' && value !== '0' && value !== 'off'
|
||||
}
|
||||
|
||||
/**
|
||||
* A GPU-decode attempt whose output stops growing for this long is killed and
|
||||
* retried with software decode, since some driver failures hang ffmpeg instead
|
||||
* of exiting. Override with SHUMAI_HW_DECODE_STALL_TIMEOUT (seconds).
|
||||
*/
|
||||
export function getHwDecodeStallTimeoutMs(): number {
|
||||
const seconds = parseFloat(process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT ?? '')
|
||||
return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : 120_000
|
||||
}
|
||||
|
||||
function getPathSize(target: string): number {
|
||||
try {
|
||||
const stat = fs.statSync(target)
|
||||
if (!stat.isDirectory()) {
|
||||
return stat.size
|
||||
}
|
||||
return fs
|
||||
.readdirSync(target)
|
||||
.reduce((total, entry) => total + getPathSize(path.join(target, entry)), 0)
|
||||
} catch {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRotation(rotation: number): number {
|
||||
return ((Math.round(rotation) % 360) + 360) % 360
|
||||
}
|
||||
|
||||
interface OrientationOptions {
|
||||
/** True when a RAW container orientation is applied by hand, so sharp must not also auto-orient. */
|
||||
rawOrientationApplied?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Sharp input options. The webp output drops the EXIF orientation tag, so camera portraits would
|
||||
* come out sideways unless sharp applies it; skip that when the RAW orientation is already applied,
|
||||
* otherwise the image would rotate twice.
|
||||
*/
|
||||
function orientedSharpOptions({ rawOrientationApplied = false }: OrientationOptions = {}) {
|
||||
return { limitInputPixels: false, autoOrient: !rawOrientationApplied } as const
|
||||
}
|
||||
|
||||
/** Width and height as displayed: sharp reports the EXIF-oriented size under `autoOrient`. */
|
||||
function displayedDimensions(
|
||||
metadata: SharpMetadata,
|
||||
{ rawOrientationApplied = false }: OrientationOptions = {},
|
||||
): Pick<SharpMetadata, 'width' | 'height'> {
|
||||
return rawOrientationApplied ? metadata : (metadata.autoOrient ?? metadata)
|
||||
}
|
||||
|
||||
export function parseBitrateKbps(bitrate: string | number): number {
|
||||
if (typeof bitrate === 'number') {
|
||||
return Math.round(bitrate / 1000)
|
||||
@@ -784,7 +932,7 @@ export class TranscodeService {
|
||||
}
|
||||
}
|
||||
|
||||
// RAW branch — extract embedded JPEG for metadata
|
||||
// RAW branch — extract embedded JPEG or decode RAW for metadata
|
||||
if (typeof input === 'string' && isRawImage(input)) {
|
||||
const extracted = await extractAndValidateRawPreview(input)
|
||||
if (extracted) {
|
||||
@@ -793,8 +941,10 @@ export class TranscodeService {
|
||||
extracted.orientation !== undefined &&
|
||||
extracted.orientation >= 5 &&
|
||||
extracted.orientation <= 8
|
||||
const originalWidth = isSwapped ? extracted.height : extracted.width
|
||||
const originalHeight = isSwapped ? extracted.width : extracted.height
|
||||
const sourceWidth = extracted.rawWidth ?? extracted.width
|
||||
const sourceHeight = extracted.rawHeight ?? extracted.height
|
||||
const originalWidth = isSwapped ? sourceHeight : sourceWidth
|
||||
const originalHeight = isSwapped ? sourceWidth : sourceHeight
|
||||
|
||||
return {
|
||||
originalWidth,
|
||||
@@ -827,9 +977,11 @@ export class TranscodeService {
|
||||
}
|
||||
|
||||
const metadata = await sharp(input, { limitInputPixels: false }).metadata()
|
||||
// Cameras store portraits sideways plus an EXIF orientation; report the size as displayed.
|
||||
const shown = displayedDimensions(metadata)
|
||||
return {
|
||||
originalWidth: metadata.width || 0,
|
||||
originalHeight: metadata.height || 0,
|
||||
originalWidth: shown.width || 0,
|
||||
originalHeight: shown.height || 0,
|
||||
duration: 0,
|
||||
bitRate: 0,
|
||||
frameRate: 0,
|
||||
@@ -898,8 +1050,12 @@ export class TranscodeService {
|
||||
}
|
||||
}
|
||||
|
||||
getDriDevice(driDir = '/dev/dri'): string | null {
|
||||
return this.getVaapiDevice(driDir)
|
||||
}
|
||||
|
||||
getVaapiDevice(driDir = '/dev/dri'): string | null {
|
||||
return getVaapiDevice(driDir)
|
||||
return getDriDevice(driDir)
|
||||
}
|
||||
|
||||
async isEncoderUsable(encoder: string): Promise<boolean> {
|
||||
@@ -1007,6 +1163,31 @@ export class TranscodeService {
|
||||
async transcodeVideo(params: TranscodeVideoParams): Promise<void> {
|
||||
const encoder = await this.selectH264Encoder(params.hardwareAcceleration)
|
||||
|
||||
if (await this.canUseHwDecode(params, encoder)) {
|
||||
try {
|
||||
await this.executeFfmpegVideoTranscode(params, encoder, { hwDecode: true })
|
||||
return
|
||||
} catch (err) {
|
||||
if (this.isAbortError(err, params.signal)) {
|
||||
throw err
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
{
|
||||
err,
|
||||
encoder: encoder.name,
|
||||
inputFile: params.inputFile,
|
||||
outputFile: params.outputFile,
|
||||
width: params.width,
|
||||
height: params.height,
|
||||
},
|
||||
'Hardware decode failed; retrying with software decode and hardware encode',
|
||||
)
|
||||
|
||||
this.removeFileIfExists(params.outputFile)
|
||||
}
|
||||
}
|
||||
|
||||
if (encoder.name !== 'libx264') {
|
||||
try {
|
||||
await this.executeFfmpegVideoTranscode(params, encoder)
|
||||
@@ -1028,13 +1209,7 @@ export class TranscodeService {
|
||||
'Hardware video transcoding failed; falling back to software transcode (libx264)',
|
||||
)
|
||||
|
||||
if (fs.existsSync(params.outputFile)) {
|
||||
try {
|
||||
fs.unlinkSync(params.outputFile)
|
||||
} catch {
|
||||
// ignore unlink errors
|
||||
}
|
||||
}
|
||||
this.removeFileIfExists(params.outputFile)
|
||||
|
||||
await this.executeFfmpegVideoTranscode(params, H264_ENCODER_CONFIGS.libx264)
|
||||
return
|
||||
@@ -1044,9 +1219,217 @@ export class TranscodeService {
|
||||
await this.executeFfmpegVideoTranscode(params, encoder)
|
||||
}
|
||||
|
||||
/**
|
||||
* GPU decode + scale is attempted only for plain proxies: watermark overlays,
|
||||
* HDR sources and rotated sources keep the software decode path. (ffmpeg
|
||||
* cannot auto-insert its rotation filter on GPU frames, so a rotated phone
|
||||
* clip would otherwise come out sideways rather than fail.) Sources the GPU
|
||||
* cannot decode (e.g. HEVC/10-bit on older Intel iGPUs) fail fast and are
|
||||
* retried with software decode by the caller.
|
||||
*/
|
||||
private async canUseHwDecode(
|
||||
params: Pick<
|
||||
TranscodeVideoParams,
|
||||
| 'inputFile'
|
||||
| 'streamIndex'
|
||||
| 'overlayFile'
|
||||
| 'sourceIsHdr'
|
||||
| 'sourceHdrType'
|
||||
| 'sourceRotation'
|
||||
>,
|
||||
encoder: EncoderConfig,
|
||||
): Promise<boolean> {
|
||||
if (!HW_DECODE_CONFIGS[encoder.name] || !isHwDecodeEnabled()) {
|
||||
return false
|
||||
}
|
||||
if (params.overlayFile) {
|
||||
return false
|
||||
}
|
||||
const isSourceHdr =
|
||||
params.sourceIsHdr ||
|
||||
params.sourceHdrType === 'pq' ||
|
||||
params.sourceHdrType === 'hlg' ||
|
||||
params.sourceHdrType === 'dovi_p5'
|
||||
if (isSourceHdr) {
|
||||
return false
|
||||
}
|
||||
if (encoder.name === 'h264_vaapi' && !this.getDriDevice()) {
|
||||
return false
|
||||
}
|
||||
if (encoder.name === 'h264_qsv' && process.platform === 'linux' && !this.getDriDevice()) {
|
||||
return false
|
||||
}
|
||||
const rotation =
|
||||
params.sourceRotation !== undefined
|
||||
? normalizeRotation(params.sourceRotation)
|
||||
: await this.getVideoRotation(params.inputFile, params.streamIndex)
|
||||
return rotation === 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Rotation of the source video stream in degrees (0, 90, 180 or 270), from
|
||||
* the display matrix or the legacy `rotate` tag. Returns null if it cannot be
|
||||
* determined.
|
||||
*/
|
||||
async getVideoRotation(inputFile: string, streamIndex?: number): Promise<number | null> {
|
||||
try {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
'-select_streams',
|
||||
streamIndex !== undefined ? String(streamIndex) : 'V:0',
|
||||
'-show_entries',
|
||||
'stream_side_data=rotation:stream_tags=rotate',
|
||||
'-of',
|
||||
'json',
|
||||
inputFile,
|
||||
])
|
||||
const stream = JSON.parse(stdout)?.streams?.[0]
|
||||
if (!stream) {
|
||||
return null
|
||||
}
|
||||
let rotation = 0
|
||||
const sideData = Array.isArray(stream.side_data_list) ? stream.side_data_list : []
|
||||
const matrix = sideData.find((d: { rotation?: unknown }) => typeof d.rotation === 'number')
|
||||
if (matrix) {
|
||||
rotation = matrix.rotation
|
||||
} else if (stream.tags?.rotate !== undefined) {
|
||||
rotation = parseFloat(stream.tags.rotate) || 0
|
||||
}
|
||||
return normalizeRotation(rotation)
|
||||
} catch (err) {
|
||||
logger.debug({ err, inputFile }, 'Failed to probe video rotation')
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private removeFileIfExists(filePath: string): void {
|
||||
if (fs.existsSync(filePath)) {
|
||||
try {
|
||||
fs.unlinkSync(filePath)
|
||||
} catch {
|
||||
// ignore unlink errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inputs, video filter graph and `-map [vout]`, shared by the MP4 and HLS
|
||||
* paths: GPU decode + GPU scale when `hwDecodeConfig` is set, otherwise
|
||||
* software decode + scale (+ upload for VAAPI encode).
|
||||
*/
|
||||
private buildVideoInputArgs(
|
||||
params: Pick<
|
||||
TranscodeVideoParams,
|
||||
'inputFile' | 'overlayFile' | 'streamIndex' | 'width' | 'height'
|
||||
>,
|
||||
options: {
|
||||
isVaapi: boolean
|
||||
driDevice?: string
|
||||
hwDecodeConfig?: HardwareDecodeConfig
|
||||
frameRate?: number | string
|
||||
},
|
||||
): string[] {
|
||||
const { isVaapi, driDevice, hwDecodeConfig, frameRate } = options
|
||||
const args: string[] = []
|
||||
|
||||
if (isVaapi && driDevice) {
|
||||
args.push('-init_hw_device', `vaapi=accel:${driDevice}`, '-filter_hw_device', 'accel')
|
||||
}
|
||||
if (hwDecodeConfig) {
|
||||
args.push(...resolveHwDecodeInputArgs(hwDecodeConfig, driDevice))
|
||||
}
|
||||
|
||||
args.push('-i', params.inputFile)
|
||||
const baseScale = hwDecodeConfig
|
||||
? hwDecodeConfig.scaleFilter(params.width, params.height)
|
||||
: `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
|
||||
|
||||
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
|
||||
|
||||
let filterComplex: string
|
||||
if (params.overlayFile) {
|
||||
args.push('-i', params.overlayFile)
|
||||
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
} else {
|
||||
filterComplex = `[${vPad}]${baseScale}`
|
||||
}
|
||||
|
||||
if (frameRate) {
|
||||
filterComplex += `,fps=${frameRate}`
|
||||
}
|
||||
if (isVaapi && !hwDecodeConfig) {
|
||||
filterComplex += ',format=nv12,hwupload=extra_hw_frames=64'
|
||||
}
|
||||
filterComplex += '[vout]'
|
||||
|
||||
args.push('-filter_complex', filterComplex, '-map', '[vout]')
|
||||
if (hwDecodeConfig) {
|
||||
args.push(...hwDecodeConfig.outputArgs)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs ffmpeg. With `stallWatchPath`, ffmpeg is killed (SIGKILL; hung VAAPI
|
||||
* processes have been seen to ignore SIGTERM) if that output file/directory
|
||||
* stops growing for {@link getHwDecodeStallTimeoutMs}, and the call rejects
|
||||
* with a regular error so the caller can fall back.
|
||||
*/
|
||||
private async runFfmpeg(
|
||||
args: string[],
|
||||
options: { signal?: AbortSignal; stallWatchPath?: string } = {},
|
||||
): Promise<void> {
|
||||
const ffmpegArgs = ['-y', '-loglevel', 'warning', ...args]
|
||||
|
||||
if (!options.stallWatchPath) {
|
||||
if (options.signal) {
|
||||
await execFileAsync('ffmpeg', ffmpegArgs, { signal: options.signal })
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ffmpegArgs)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const stallTimeoutMs = getHwDecodeStallTimeoutMs()
|
||||
const stall = new AbortController()
|
||||
const signal = options.signal ? AbortSignal.any([options.signal, stall.signal]) : stall.signal
|
||||
const watchPath = options.stallWatchPath
|
||||
let lastSize = -1
|
||||
let lastProgressAt = Date.now()
|
||||
const timer = setInterval(
|
||||
() => {
|
||||
const size = getPathSize(watchPath)
|
||||
if (size !== lastSize) {
|
||||
lastSize = size
|
||||
lastProgressAt = Date.now()
|
||||
} else if (Date.now() - lastProgressAt >= stallTimeoutMs) {
|
||||
stall.abort()
|
||||
}
|
||||
},
|
||||
Math.min(5000, stallTimeoutMs),
|
||||
)
|
||||
timer.unref?.()
|
||||
|
||||
try {
|
||||
await execFileAsync('ffmpeg', ffmpegArgs, { signal, killSignal: 'SIGKILL' })
|
||||
} catch (err) {
|
||||
if (stall.signal.aborted && !options.signal?.aborted) {
|
||||
throw new Error(
|
||||
`ffmpeg made no progress for ${Math.round(stallTimeoutMs / 1000)}s and was killed`,
|
||||
{ cause: err },
|
||||
)
|
||||
}
|
||||
throw err
|
||||
} finally {
|
||||
clearInterval(timer)
|
||||
}
|
||||
}
|
||||
|
||||
private async executeFfmpegVideoTranscode(
|
||||
params: TranscodeVideoParams,
|
||||
encoder: EncoderConfig,
|
||||
options: { hwDecode?: boolean } = {},
|
||||
): Promise<void> {
|
||||
const isSourceHdr =
|
||||
params.sourceIsHdr ||
|
||||
@@ -1056,7 +1439,9 @@ export class TranscodeService {
|
||||
const isHdrOutput = Boolean(params.hdr)
|
||||
|
||||
const isVaapi = encoder.name === 'h264_vaapi'
|
||||
const vaapiDevice = isVaapi ? (this.getVaapiDevice() ?? '/dev/dri/renderD128') : undefined
|
||||
const driDevice = this.getDriDevice() ?? (isVaapi ? '/dev/dri/renderD128' : undefined)
|
||||
const hwDecodeConfig = options.hwDecode ? HW_DECODE_CONFIGS[encoder.name] : undefined
|
||||
const hwDecode = Boolean(hwDecodeConfig)
|
||||
|
||||
logger.info(
|
||||
{
|
||||
@@ -1064,39 +1449,19 @@ export class TranscodeService {
|
||||
hardwareAcceleration: params.hardwareAcceleration ?? 'off',
|
||||
inputFile: params.inputFile,
|
||||
outputFile: params.outputFile,
|
||||
vaapiDevice,
|
||||
vaapiDevice: isVaapi ? driDevice : undefined,
|
||||
driDevice,
|
||||
hwDecode,
|
||||
},
|
||||
'Starting video transcoding',
|
||||
)
|
||||
|
||||
let filterComplex: string
|
||||
const args: string[] = []
|
||||
|
||||
if (isVaapi && vaapiDevice) {
|
||||
args.push('-init_hw_device', `vaapi=accel:${vaapiDevice}`, '-filter_hw_device', 'accel')
|
||||
}
|
||||
|
||||
args.push('-i', params.inputFile)
|
||||
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
|
||||
|
||||
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
|
||||
|
||||
if (params.overlayFile) {
|
||||
args.push('-i', params.overlayFile)
|
||||
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
} else {
|
||||
filterComplex = `[${vPad}]${baseScale}`
|
||||
}
|
||||
|
||||
if (params.frameRate) {
|
||||
filterComplex += `,fps=${params.frameRate}`
|
||||
}
|
||||
if (isVaapi) {
|
||||
filterComplex += ',format=nv12,hwupload=extra_hw_frames=64'
|
||||
}
|
||||
filterComplex += '[vout]'
|
||||
|
||||
args.push('-filter_complex', filterComplex, '-map', '[vout]')
|
||||
const args = this.buildVideoInputArgs(params, {
|
||||
isVaapi,
|
||||
driDevice,
|
||||
hwDecodeConfig,
|
||||
frameRate: params.frameRate,
|
||||
})
|
||||
|
||||
if (params.frameRate) {
|
||||
let calculatedFps: number
|
||||
@@ -1171,6 +1536,18 @@ export class TranscodeService {
|
||||
'-minrate',
|
||||
`${minKbps}k`,
|
||||
)
|
||||
} else if (encoder.name === 'h264_rkmpp') {
|
||||
if (params.videoBitrate) {
|
||||
args.push('-b:v', params.videoBitrate)
|
||||
} else {
|
||||
const { maxrate } = calculateMaxBitrate(
|
||||
params.height,
|
||||
params.width,
|
||||
params.sourceVideoBitrate,
|
||||
params.frameRate,
|
||||
)
|
||||
args.push('-b:v', maxrate)
|
||||
}
|
||||
} else {
|
||||
if (params.videoBitrate) {
|
||||
args.push('-b:v', params.videoBitrate)
|
||||
@@ -1185,7 +1562,7 @@ export class TranscodeService {
|
||||
}
|
||||
}
|
||||
|
||||
if (!isVaapi) {
|
||||
if (!hwDecode && !isVaapi) {
|
||||
args.push('-pix_fmt', 'yuv420p')
|
||||
}
|
||||
|
||||
@@ -1208,18 +1585,40 @@ export class TranscodeService {
|
||||
|
||||
args.push('-movflags', '+faststart', '-max_muxing_queue_size', '1024', params.outputFile)
|
||||
|
||||
if (params.signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
|
||||
signal: params.signal,
|
||||
})
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
await this.runFfmpeg(args, {
|
||||
signal: params.signal,
|
||||
stallWatchPath: hwDecode ? params.outputFile : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
async transcodeHlsRendition(params: TranscodeHlsRenditionParams): Promise<void> {
|
||||
const encoder = await this.selectH264Encoder(params.hardwareAcceleration)
|
||||
|
||||
if (await this.canUseHwDecode(params, encoder)) {
|
||||
try {
|
||||
await this.executeFfmpegHlsTranscode(params, encoder, { hwDecode: true })
|
||||
return
|
||||
} catch (err) {
|
||||
if (this.isAbortError(err, params.signal)) {
|
||||
throw err
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
{
|
||||
err,
|
||||
encoder: encoder.name,
|
||||
inputFile: params.inputFile,
|
||||
outputDir: params.outputDir,
|
||||
width: params.width,
|
||||
height: params.height,
|
||||
},
|
||||
'Hardware decode failed for HLS; retrying with software decode and hardware encode',
|
||||
)
|
||||
|
||||
this.removeDirIfExists(params.outputDir)
|
||||
}
|
||||
}
|
||||
|
||||
if (encoder.name !== 'libx264') {
|
||||
try {
|
||||
await this.executeFfmpegHlsTranscode(params, encoder)
|
||||
@@ -1241,13 +1640,7 @@ export class TranscodeService {
|
||||
'Hardware HLS transcoding failed; falling back to software transcode (libx264)',
|
||||
)
|
||||
|
||||
if (fs.existsSync(params.outputDir)) {
|
||||
try {
|
||||
fs.rmSync(params.outputDir, { recursive: true, force: true })
|
||||
} catch {
|
||||
// ignore unlink errors
|
||||
}
|
||||
}
|
||||
this.removeDirIfExists(params.outputDir)
|
||||
|
||||
await this.executeFfmpegHlsTranscode(params, H264_ENCODER_CONFIGS.libx264)
|
||||
return
|
||||
@@ -1257,9 +1650,20 @@ export class TranscodeService {
|
||||
await this.executeFfmpegHlsTranscode(params, encoder)
|
||||
}
|
||||
|
||||
private removeDirIfExists(dirPath: string): void {
|
||||
if (fs.existsSync(dirPath)) {
|
||||
try {
|
||||
fs.rmSync(dirPath, { recursive: true, force: true })
|
||||
} catch {
|
||||
// ignore unlink errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async executeFfmpegHlsTranscode(
|
||||
params: TranscodeHlsRenditionParams,
|
||||
encoder: EncoderConfig,
|
||||
options: { hwDecode?: boolean } = {},
|
||||
): Promise<void> {
|
||||
fs.mkdirSync(params.outputDir, { recursive: true })
|
||||
|
||||
@@ -1271,7 +1675,9 @@ export class TranscodeService {
|
||||
const isHdrOutput = Boolean(params.hdr)
|
||||
|
||||
const isVaapi = encoder.name === 'h264_vaapi'
|
||||
const vaapiDevice = isVaapi ? (this.getVaapiDevice() ?? '/dev/dri/renderD128') : undefined
|
||||
const driDevice = this.getDriDevice() ?? (isVaapi ? '/dev/dri/renderD128' : undefined)
|
||||
const hwDecodeConfig = options.hwDecode ? HW_DECODE_CONFIGS[encoder.name] : undefined
|
||||
const hwDecode = Boolean(hwDecodeConfig)
|
||||
|
||||
const segmentDuration = params.segmentDuration || 4
|
||||
let calculatedFps = 30
|
||||
@@ -1302,37 +1708,17 @@ export class TranscodeService {
|
||||
height: params.height,
|
||||
segmentDuration,
|
||||
gopSize,
|
||||
hwDecode,
|
||||
},
|
||||
'Starting HLS rendition transcoding',
|
||||
)
|
||||
|
||||
let filterComplex: string
|
||||
const args: string[] = []
|
||||
|
||||
if (isVaapi && vaapiDevice) {
|
||||
args.push('-init_hw_device', `vaapi=accel:${vaapiDevice}`, '-filter_hw_device', 'accel')
|
||||
}
|
||||
|
||||
args.push('-i', params.inputFile)
|
||||
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
|
||||
|
||||
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
|
||||
|
||||
if (params.overlayFile) {
|
||||
args.push('-i', params.overlayFile)
|
||||
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
} else {
|
||||
filterComplex = `[${vPad}]${baseScale}`
|
||||
}
|
||||
|
||||
filterComplex += `,fps=${calculatedFps}`
|
||||
|
||||
if (isVaapi) {
|
||||
filterComplex += ',format=nv12,hwupload=extra_hw_frames=64'
|
||||
}
|
||||
filterComplex += '[vout]'
|
||||
|
||||
args.push('-filter_complex', filterComplex, '-map', '[vout]')
|
||||
const args = this.buildVideoInputArgs(params, {
|
||||
isVaapi,
|
||||
driDevice,
|
||||
hwDecodeConfig,
|
||||
frameRate: calculatedFps,
|
||||
})
|
||||
|
||||
if (!params.disableAudio) {
|
||||
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
|
||||
@@ -1396,7 +1782,7 @@ export class TranscodeService {
|
||||
maxKbps = parseInt(maxrate, 10)
|
||||
}
|
||||
const targetKbps = Math.max(25, Math.ceil(maxKbps / 1.45))
|
||||
args.push('-b:v', `${targetKbps}k`, '-maxrate', `${maxKbps}k`)
|
||||
args.push('-b:v', `${targetKbps}k`, '-maxrate', `${maxKbps}k`, '-idr_interval', '0')
|
||||
} else if (encoder.name === 'h264_nvenc') {
|
||||
let maxrateStr: string
|
||||
let bufsizeStr: string
|
||||
@@ -1427,7 +1813,23 @@ export class TranscodeService {
|
||||
'1',
|
||||
'-temporal-aq',
|
||||
'1',
|
||||
'-forced-idr',
|
||||
'1',
|
||||
)
|
||||
} else if (encoder.name === 'h264_rkmpp') {
|
||||
let maxKbps: number
|
||||
if (params.videoBitrate) {
|
||||
maxKbps = parseBitrateKbps(params.videoBitrate)
|
||||
} else {
|
||||
const { maxrate } = calculateMaxBitrate(
|
||||
params.height,
|
||||
params.width,
|
||||
params.sourceVideoBitrate,
|
||||
calculatedFps,
|
||||
)
|
||||
maxKbps = parseInt(maxrate, 10)
|
||||
}
|
||||
args.push('-b:v', `${maxKbps}k`)
|
||||
} else {
|
||||
let maxrateStr: string
|
||||
let bufsizeStr: string
|
||||
@@ -1448,7 +1850,7 @@ export class TranscodeService {
|
||||
args.push('-crf', '23', '-maxrate', maxrateStr, '-bufsize', bufsizeStr)
|
||||
}
|
||||
|
||||
if (!isVaapi) {
|
||||
if (!hwDecode && !isVaapi) {
|
||||
args.push('-pix_fmt', 'yuv420p')
|
||||
}
|
||||
|
||||
@@ -1506,13 +1908,10 @@ export class TranscodeService {
|
||||
playlistFile,
|
||||
)
|
||||
|
||||
if (params.signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
|
||||
signal: params.signal,
|
||||
})
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
await this.runFfmpeg(args, {
|
||||
signal: params.signal,
|
||||
stallWatchPath: hwDecode ? params.outputDir : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
async transcodeImage(
|
||||
@@ -1545,16 +1944,20 @@ export class TranscodeService {
|
||||
input = Buffer.from(await resp.arrayBuffer())
|
||||
}
|
||||
|
||||
// RAW branch — extract embedded JPEG to temporary file + orientation, then treat as normal image
|
||||
// RAW branch — extract embedded preview or decode RAW to temporary file + orientation, then treat as normal image
|
||||
let rawOrientation: number | undefined
|
||||
let rawOutputUpright = false
|
||||
let rawCleanup: (() => void) | null = null
|
||||
if (typeof input === 'string' && isRawImage(input)) {
|
||||
const extracted = await extractAndValidateRawPreview(input)
|
||||
if (!extracted) {
|
||||
throw new Error(`Cannot generate preview for RAW file: no usable embedded JPEG in ${input}`)
|
||||
throw new Error(
|
||||
`Cannot generate preview for RAW file: no usable embedded preview or decoded image in ${input}`,
|
||||
)
|
||||
}
|
||||
input = extracted.previewPath
|
||||
rawOrientation = extracted.orientation
|
||||
rawOutputUpright = extracted.orientationApplied === true
|
||||
rawCleanup = extracted.cleanup
|
||||
}
|
||||
|
||||
@@ -1563,11 +1966,17 @@ export class TranscodeService {
|
||||
let targetW = width > 0 ? Math.min(width, WEBP_MAX_DIMENSION) : WEBP_MAX_DIMENSION
|
||||
let targetH = height && height > 0 ? Math.min(height, WEBP_MAX_DIMENSION) : WEBP_MAX_DIMENSION
|
||||
|
||||
const sharpInstance = sharp(input, { limitInputPixels: false })
|
||||
// RAW previews are oriented from the container EXIF below, so only auto-orient other
|
||||
// inputs: the webp output drops the tag, which would leave camera portraits sideways.
|
||||
// dcraw_emu output is already upright, so it is never auto-oriented (no double rotation).
|
||||
const rawOrientationApplied = rawOrientation !== undefined || rawOutputUpright
|
||||
const sharpInstance = sharp(input, orientedSharpOptions({ rawOrientationApplied }))
|
||||
|
||||
if (isPreview) {
|
||||
try {
|
||||
const meta = await sharpInstance.metadata()
|
||||
const meta = displayedDimensions(await sharpInstance.metadata(), {
|
||||
rawOrientationApplied,
|
||||
})
|
||||
if (meta.width && meta.height) {
|
||||
const isSwapped =
|
||||
rawOrientation !== undefined && rawOrientation >= 5 && rawOrientation <= 8
|
||||
@@ -1626,7 +2035,7 @@ export class TranscodeService {
|
||||
|
||||
// Apply EXIF orientation from the RAW container to the extracted buffer.
|
||||
// The buffer itself often lacks orientation EXIF, so Sharp won't auto-rotate.
|
||||
// For non-RAW images, Sharp auto-rotates from the image's own EXIF.
|
||||
// For non-RAW images, autoOrient above applies the image's own EXIF.
|
||||
if (rawOrientation && EXIF_ORIENTATION_TO_ROTATION[rawOrientation]) {
|
||||
const { angle, flip, flop } = EXIF_ORIENTATION_TO_ROTATION[rawOrientation]
|
||||
if (angle) sharpInstance.rotate(angle)
|
||||
@@ -2642,12 +3051,15 @@ export class TranscodeService {
|
||||
return imageBuffer
|
||||
}
|
||||
|
||||
const meta = await sharp(imageBuffer, { limitInputPixels: false }).metadata()
|
||||
// Annotations are drawn in the displayed (EXIF-oriented) coordinate space.
|
||||
const meta = displayedDimensions(
|
||||
await sharp(imageBuffer, { limitInputPixels: false }).metadata(),
|
||||
)
|
||||
const width = meta.width || 1920
|
||||
const height = meta.height || 1080
|
||||
|
||||
const svgStr = renderAnnotationsToSvg(width, height, annotations)
|
||||
return await sharp(imageBuffer, { limitInputPixels: false })
|
||||
return await sharp(imageBuffer, orientedSharpOptions())
|
||||
.composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }])
|
||||
.toColorspace('srgb')
|
||||
.resize(16383, 16383, { fit: 'inside', withoutEnlargement: true })
|
||||
@@ -2734,13 +3146,83 @@ export class TranscodeService {
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<void> {
|
||||
await sharp(inputPath, { limitInputPixels: false })
|
||||
await sharp(inputPath, orientedSharpOptions())
|
||||
.toColorspace('srgb')
|
||||
.resize(width, height, { fit: 'inside' })
|
||||
.composite([{ input: overlayPngBuffer }])
|
||||
.webp({ quality: 90 })
|
||||
.toFile(outputPath)
|
||||
}
|
||||
|
||||
async convertImageToWebp(imageBuffer: Buffer, quality = 85): Promise<Buffer> {
|
||||
return await sharp(imageBuffer, { limitInputPixels: false })
|
||||
.toColorspace('srgb')
|
||||
.webp({ quality })
|
||||
.toBuffer()
|
||||
}
|
||||
|
||||
async generate3dSprite(
|
||||
inputFile: string,
|
||||
outputSprite: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
throw new Error('Sprite generation cancelled')
|
||||
}
|
||||
const spriteFps = 100 / 4
|
||||
const filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
|
||||
const args = [
|
||||
'-i',
|
||||
inputFile,
|
||||
'-filter_complex',
|
||||
filterComplex,
|
||||
'-map',
|
||||
'[sprite_out]',
|
||||
'-frames:v',
|
||||
'1',
|
||||
'-c:v',
|
||||
'libwebp',
|
||||
'-q:v',
|
||||
'75',
|
||||
outputSprite,
|
||||
]
|
||||
if (signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
}
|
||||
|
||||
async generate3dViews(
|
||||
inputFile: string,
|
||||
outputViews: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
throw new Error('3D views generation cancelled')
|
||||
}
|
||||
const filterComplex = `[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]`
|
||||
const args = [
|
||||
'-i',
|
||||
inputFile,
|
||||
'-filter_complex',
|
||||
filterComplex,
|
||||
'-map',
|
||||
'[views_out]',
|
||||
'-frames:v',
|
||||
'1',
|
||||
'-c:v',
|
||||
'libwebp',
|
||||
'-q:v',
|
||||
'85',
|
||||
outputViews,
|
||||
]
|
||||
if (signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function renderAnnotationsToSvg(
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import * as os from 'os'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { TurntableService } from './turntable'
|
||||
|
||||
describe('TurntableService', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
const originalEnv = process.env
|
||||
let tmpDir: string
|
||||
let testFilePath: string
|
||||
|
||||
beforeEach(() => {
|
||||
process.env = { ...originalEnv }
|
||||
delete process.env.TURNTABLE_RENDERER_URL
|
||||
delete process.env.TURNTABLE_RENDERER_BASIC_AUTH_USERNAME
|
||||
delete process.env.TURNTABLE_RENDERER_BASIC_AUTH_PASSWORD
|
||||
delete process.env.TURNTABLE_RENDERER_USERNAME
|
||||
delete process.env.TURNTABLE_RENDERER_PASSWORD
|
||||
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'turntable-test-'))
|
||||
testFilePath = path.join(tmpDir, 'model.glb')
|
||||
fs.writeFileSync(testFilePath, 'fake glb content')
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('resolveConfig', () => {
|
||||
it('returns isEnv: false when neither env nor team settings are present', async () => {
|
||||
const service = new TurntableService()
|
||||
const config = await service.resolveConfig()
|
||||
expect(config.url).toBeUndefined()
|
||||
expect(config.isEnv).toBe(false)
|
||||
})
|
||||
|
||||
it('resolves env configuration with highest priority', async () => {
|
||||
process.env.TURNTABLE_RENDERER_URL = 'http://localhost:3000/'
|
||||
process.env.TURNTABLE_RENDERER_BASIC_AUTH_USERNAME = 'admin'
|
||||
process.env.TURNTABLE_RENDERER_BASIC_AUTH_PASSWORD = 'secret'
|
||||
|
||||
const service = new TurntableService()
|
||||
const config = await service.resolveConfig('any-team-id')
|
||||
|
||||
expect(config.url).toBe('http://localhost:3000')
|
||||
expect(config.username).toBe('admin')
|
||||
expect(config.password).toBe('secret')
|
||||
expect(config.isEnv).toBe(true)
|
||||
})
|
||||
|
||||
it('falls back to team DB settings when env is not set', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'Test Team 3D',
|
||||
settings: {
|
||||
turntable: {
|
||||
url: 'http://custom-turntable:4000',
|
||||
username: 'teamUser',
|
||||
password: 'teamPassword',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const service = new TurntableService()
|
||||
const config = await service.resolveConfig(team.id)
|
||||
|
||||
expect(config.url).toBe('http://custom-turntable:4000')
|
||||
expect(config.username).toBe('teamUser')
|
||||
expect(config.password).toBe('teamPassword')
|
||||
expect(config.isEnv).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('isAvailable', () => {
|
||||
it('returns false if URL is not configured', async () => {
|
||||
const service = new TurntableService()
|
||||
expect(await service.isAvailable()).toBe(false)
|
||||
})
|
||||
|
||||
it('returns true when health check returns 200', async () => {
|
||||
process.env.TURNTABLE_RENDERER_URL = 'http://turntable:3000'
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
expect(await service.isAvailable()).toBe(true)
|
||||
})
|
||||
|
||||
it('returns false when health check fails or throws', async () => {
|
||||
process.env.TURNTABLE_RENDERER_URL = 'http://turntable:3000'
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('Connection refused'))
|
||||
|
||||
const service = new TurntableService()
|
||||
expect(await service.isAvailable()).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('testConnection', () => {
|
||||
it('returns error when url is empty', async () => {
|
||||
const service = new TurntableService()
|
||||
const result = await service.testConnection({})
|
||||
expect(result.ok).toBe(false)
|
||||
expect(result.error).toBe('URL is required')
|
||||
})
|
||||
|
||||
it('returns version information upon successful connection', async () => {
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ version: '0.1.0', blender: '5.2.2' }),
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
const result = await service.testConnection({ url: 'http://localhost:3000' })
|
||||
|
||||
expect(result.ok).toBe(true)
|
||||
expect(result.version).toBe('0.1.0')
|
||||
expect(result.blender).toBe('5.2.2')
|
||||
})
|
||||
})
|
||||
|
||||
describe('uploadFile', () => {
|
||||
it('posts file buffer to /v1/files and returns file info', async () => {
|
||||
process.env.TURNTABLE_RENDERER_URL = 'http://turntable:3000'
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 201,
|
||||
json: async () => ({ id: 'f_test123', name: 'model.glb', size: 100 }),
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
const res = await service.uploadFile(testFilePath, 'model.glb')
|
||||
|
||||
expect(res.id).toBe('f_test123')
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'http://turntable:3000/v1/files',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('startRender & getTaskStatus & downloadFile & deleteTask', () => {
|
||||
beforeEach(() => {
|
||||
process.env.TURNTABLE_RENDERER_URL = 'http://turntable:3000'
|
||||
})
|
||||
|
||||
it('startRender calls /v1/render and returns sync response', async () => {
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
taskId: 't_test456',
|
||||
status: 'queued',
|
||||
poster: {
|
||||
fileId: 'f_poster',
|
||||
size: 200,
|
||||
contentType: 'image/png',
|
||||
width: 1080,
|
||||
height: 1080,
|
||||
},
|
||||
}),
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
const res = await service.startRender('f_input')
|
||||
expect(res.taskId).toBe('t_test456')
|
||||
expect(res.poster.fileId).toBe('f_poster')
|
||||
})
|
||||
|
||||
it('getTaskStatus queries /v1/render/tasks/:id', async () => {
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
taskId: 't_test456',
|
||||
status: 'completed',
|
||||
video: {
|
||||
fileId: 'f_video',
|
||||
size: 1000,
|
||||
contentType: 'video/mp4',
|
||||
width: 1080,
|
||||
height: 1080,
|
||||
},
|
||||
}),
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
const res = await service.getTaskStatus('t_test456')
|
||||
expect(res.status).toBe('completed')
|
||||
expect(res.video?.fileId).toBe('f_video')
|
||||
})
|
||||
|
||||
it('downloadFile fetches /v1/files/:id and writes to destination', async () => {
|
||||
const destPath = path.join(tmpDir, 'downloaded.mp4')
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () => Buffer.from('video data').buffer,
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
await service.downloadFile('f_video', destPath)
|
||||
|
||||
expect(fs.existsSync(destPath)).toBe(true)
|
||||
expect(fs.readFileSync(destPath, 'utf8')).toBe('video data')
|
||||
})
|
||||
|
||||
it('deleteTask calls DELETE /v1/render/tasks/:id and returns boolean', async () => {
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
} as Response)
|
||||
|
||||
const service = new TurntableService()
|
||||
const deleted = await service.deleteTask('t_test456')
|
||||
|
||||
expect(deleted).toBe(true)
|
||||
expect(fetchSpy).toHaveBeenCalledWith(
|
||||
'http://turntable:3000/v1/render/tasks/t_test456',
|
||||
expect.objectContaining({
|
||||
method: 'DELETE',
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,364 @@
|
||||
import * as fs from 'fs'
|
||||
import type { TurntableSettingsResponse, UpdateTurntableSettingsRequest } from '@shumai/dtos'
|
||||
import {
|
||||
IntegrationDefinition,
|
||||
resolveIntegrationConfig,
|
||||
getIntegrationSettings,
|
||||
updateIntegrationSettings,
|
||||
resolveTestConnectionConfig,
|
||||
getIntegrationBasicAuthHeader,
|
||||
} from '@shumai/core/src/integrations/helper'
|
||||
|
||||
export const TURNTABLE_INTEGRATION_DEF: IntegrationDefinition<'turntable'> = {
|
||||
name: 'Turntable',
|
||||
settingsKey: 'turntable',
|
||||
envKeys: {
|
||||
url: ['TURNTABLE_RENDERER_URL'],
|
||||
username: ['TURNTABLE_RENDERER_BASIC_AUTH_USERNAME', 'TURNTABLE_RENDERER_USERNAME'],
|
||||
password: ['TURNTABLE_RENDERER_BASIC_AUTH_PASSWORD', 'TURNTABLE_RENDERER_PASSWORD'],
|
||||
},
|
||||
}
|
||||
|
||||
export interface TurntableConfig {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
isEnv: boolean
|
||||
}
|
||||
|
||||
export interface TurntablePosterOutput {
|
||||
fileId: string
|
||||
size: number
|
||||
contentType: string
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
export interface TurntableRenderSyncResponse {
|
||||
taskId: string
|
||||
status: 'queued' | 'rendering'
|
||||
positionInQueue?: number
|
||||
metadata?: Record<string, unknown>
|
||||
poster: TurntablePosterOutput
|
||||
}
|
||||
|
||||
export interface TurntableVideoOutput {
|
||||
fileId: string
|
||||
size: number
|
||||
contentType: string
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
export interface TurntableRenderStats {
|
||||
width: number
|
||||
height: number
|
||||
frames: number
|
||||
fps: number
|
||||
durationSeconds: number
|
||||
degreesPerFrame: number
|
||||
startAngle: number
|
||||
direction: string
|
||||
includeEndFrame: boolean
|
||||
engine: string
|
||||
elapsedMs: number
|
||||
}
|
||||
|
||||
export interface TurntableTaskResponse {
|
||||
taskId: string
|
||||
status: 'queued' | 'rendering' | 'completed' | 'failed'
|
||||
positionInQueue?: number
|
||||
createdAt: string
|
||||
startedAt?: string
|
||||
completedAt?: string
|
||||
failedAt?: string
|
||||
render?: TurntableRenderStats
|
||||
video?: TurntableVideoOutput
|
||||
error?: {
|
||||
code: string
|
||||
message: string
|
||||
details?: Record<string, unknown>
|
||||
}
|
||||
}
|
||||
|
||||
export class TurntableService {
|
||||
async resolveConfig(teamId?: string): Promise<TurntableConfig> {
|
||||
return resolveIntegrationConfig(TURNTABLE_INTEGRATION_DEF, teamId)
|
||||
}
|
||||
|
||||
private getAuthHeader(config: TurntableConfig): string | undefined {
|
||||
return getIntegrationBasicAuthHeader(config)
|
||||
}
|
||||
|
||||
async isAvailable(teamId?: string): Promise<boolean> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${config.url}/health`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(3000),
|
||||
})
|
||||
|
||||
return response.ok
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async getSettings(teamId: string): Promise<TurntableSettingsResponse> {
|
||||
return getIntegrationSettings(TURNTABLE_INTEGRATION_DEF, teamId)
|
||||
}
|
||||
|
||||
async updateSettings(
|
||||
teamId: string,
|
||||
req: UpdateTurntableSettingsRequest,
|
||||
): Promise<TurntableSettingsResponse> {
|
||||
return updateIntegrationSettings(TURNTABLE_INTEGRATION_DEF, teamId, req)
|
||||
}
|
||||
|
||||
async testConnection(
|
||||
config?: {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
},
|
||||
teamId?: string,
|
||||
): Promise<{ ok: boolean; version?: string; blender?: string; error?: string }> {
|
||||
const target = await resolveTestConnectionConfig(TURNTABLE_INTEGRATION_DEF, config, teamId)
|
||||
|
||||
if (!target.resolvedUrl) {
|
||||
return { ok: false, error: 'URL is required' }
|
||||
}
|
||||
const cleanUrl = target.resolvedUrl.replace(/\/+$/, '')
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = getIntegrationBasicAuthHeader({
|
||||
username: target.resolvedUsername,
|
||||
password: target.resolvedPassword,
|
||||
})
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${cleanUrl}/version`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(5000),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errText = await response.text().catch(() => '')
|
||||
return {
|
||||
ok: false,
|
||||
error: `Server returned status ${response.status}: ${errText || response.statusText}`,
|
||||
}
|
||||
}
|
||||
|
||||
const body = (await response.json()) as { version?: string; blender?: string }
|
||||
return {
|
||||
ok: true,
|
||||
version: body.version,
|
||||
blender: body.blender,
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err)
|
||||
return { ok: false, error: message }
|
||||
}
|
||||
}
|
||||
|
||||
async uploadFile(
|
||||
filePath: string,
|
||||
filename: string,
|
||||
teamId?: string,
|
||||
): Promise<{ id: string; name: string; size: number }> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Turntable renderer URL is not configured.')
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/octet-stream',
|
||||
'x-filename': encodeURIComponent(filename),
|
||||
}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${config.url}/v1/files`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: Bun.file(filePath),
|
||||
signal: AbortSignal.timeout(900000),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Turntable upload failed with status ${response.status}: ${errorText || response.statusText}`,
|
||||
)
|
||||
}
|
||||
|
||||
return (await response.json()) as { id: string; name: string; size: number }
|
||||
}
|
||||
|
||||
async startRender(
|
||||
fileId: string,
|
||||
options: Record<string, unknown> = {},
|
||||
teamId?: string,
|
||||
): Promise<TurntableRenderSyncResponse> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Turntable renderer URL is not configured.')
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${config.url}/v1/render`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
input: { fileId },
|
||||
options,
|
||||
}),
|
||||
signal: AbortSignal.timeout(60000),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Turntable render request failed with status ${response.status}: ${errorText || response.statusText}`,
|
||||
)
|
||||
}
|
||||
|
||||
return (await response.json()) as TurntableRenderSyncResponse
|
||||
}
|
||||
|
||||
async getTaskStatus(taskId: string, teamId?: string): Promise<TurntableTaskResponse> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Turntable renderer URL is not configured.')
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${config.url}/v1/render/tasks/${taskId}`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(10000),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Failed to fetch task status (${response.status}): ${errorText || response.statusText}`,
|
||||
)
|
||||
}
|
||||
|
||||
return (await response.json()) as TurntableTaskResponse
|
||||
}
|
||||
|
||||
async downloadFile(fileId: string, targetPath?: string, teamId?: string): Promise<Buffer> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
throw new Error('Turntable renderer URL is not configured.')
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
const response = await fetch(`${config.url}/v1/files/${fileId}`, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(120000),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Failed to download file ${fileId} (${response.status}): ${errorText || response.statusText}`,
|
||||
)
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer()
|
||||
const buffer = Buffer.from(arrayBuffer)
|
||||
if (targetPath) {
|
||||
fs.writeFileSync(targetPath, buffer)
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
async deleteTask(taskId: string, teamId?: string): Promise<boolean> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
return false
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${config.url}/v1/render/tasks/${taskId}`, {
|
||||
method: 'DELETE',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(10000),
|
||||
})
|
||||
return response.ok
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async deleteFile(fileId: string, teamId?: string): Promise<boolean> {
|
||||
const config = await this.resolveConfig(teamId)
|
||||
if (!config.url) {
|
||||
return false
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {}
|
||||
const authHeader = this.getAuthHeader(config)
|
||||
if (authHeader) {
|
||||
headers['Authorization'] = authHeader
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${config.url}/v1/files/${fileId}`, {
|
||||
method: 'DELETE',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(10000),
|
||||
})
|
||||
return response.ok
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const turntableService = new TurntableService()
|
||||
@@ -3,6 +3,7 @@ import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { uploadService } from './upload'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { AssetStatus, AssetType, TaskStatus, WorkflowTaskType } from '@shumai/db'
|
||||
|
||||
@@ -450,6 +451,161 @@ describe('UploadService', () => {
|
||||
expect(workflowTask).toBeNull()
|
||||
})
|
||||
|
||||
it('should create transcode_pdf for office documents when Gotenberg is configured in team settings without global env', async () => {
|
||||
vi.restoreAllMocks()
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
|
||||
const url = String(input)
|
||||
if (url.includes('/health')) {
|
||||
return new Response(JSON.stringify({ status: 'up' }), { status: 200 })
|
||||
}
|
||||
return new Response('Not Found', { status: 404 })
|
||||
})
|
||||
const originalEnv = process.env.GOTENBERG_URL
|
||||
delete process.env.GOTENBERG_URL
|
||||
|
||||
try {
|
||||
await prisma.team.update({
|
||||
where: { id: teamId },
|
||||
data: {
|
||||
settings: {
|
||||
gotenberg: {
|
||||
url: 'http://gotenberg.local:3000',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const task = await prisma.task.create({
|
||||
data: { creatorId: userId, total: 1, uploaded: 0, type: 'upload' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'report.docx',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key-doc-team-settings' },
|
||||
create: { key: 'test-key-doc-team-settings' },
|
||||
},
|
||||
},
|
||||
mediaType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
},
|
||||
})
|
||||
|
||||
await uploadService.confirmFileUpload(userId, task.id, { fileId: asset.id })
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_pdf },
|
||||
})
|
||||
expect(workflowTask).not.toBeNull()
|
||||
|
||||
// Test HTML document as well
|
||||
const htmlTask = await prisma.task.create({
|
||||
data: { creatorId: userId, total: 1, uploaded: 0, type: 'upload' },
|
||||
})
|
||||
const htmlAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'index.html',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key-html-team-settings' },
|
||||
create: { key: 'test-key-html-team-settings' },
|
||||
},
|
||||
},
|
||||
mediaType: 'text/html',
|
||||
},
|
||||
})
|
||||
|
||||
await uploadService.confirmFileUpload(userId, htmlTask.id, { fileId: htmlAsset.id })
|
||||
|
||||
const htmlWorkflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: htmlAsset.id, type: WorkflowTaskType.transcode_pdf },
|
||||
})
|
||||
expect(htmlWorkflowTask).not.toBeNull()
|
||||
} finally {
|
||||
if (originalEnv !== undefined) {
|
||||
process.env.GOTENBERG_URL = originalEnv
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('should confirm file upload and create transcode_3d for 3D model when turntable is available', async () => {
|
||||
vi.spyOn(turntableService, 'isAvailable').mockResolvedValue(true)
|
||||
const task = await prisma.task.create({
|
||||
data: { creatorId: userId, total: 1, uploaded: 0, type: 'upload' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key-3d-1' },
|
||||
create: { key: 'test-key-3d-1' },
|
||||
},
|
||||
},
|
||||
mediaType: 'model/gltf-binary',
|
||||
},
|
||||
})
|
||||
|
||||
await uploadService.confirmFileUpload(userId, task.id, { fileId: asset.id })
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_3d },
|
||||
})
|
||||
expect(workflowTask).toBeDefined()
|
||||
expect(workflowTask?.payload).toEqual({
|
||||
projectId: projectId,
|
||||
transcode: {
|
||||
sprite: true,
|
||||
poster: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('should mark asset processed without creating transcode_3d for 3D model when turntable is unavailable', async () => {
|
||||
vi.spyOn(turntableService, 'isAvailable').mockResolvedValue(false)
|
||||
const task = await prisma.task.create({
|
||||
data: { creatorId: userId, total: 1, uploaded: 0, type: 'upload' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: AssetType.file,
|
||||
project: { connect: { id: projectId } },
|
||||
parent: { connect: { id: parentId } },
|
||||
status: AssetStatus.uploading,
|
||||
storageKey: {
|
||||
connectOrCreate: {
|
||||
where: { key: 'test-key-3d-2' },
|
||||
create: { key: 'test-key-3d-2' },
|
||||
},
|
||||
},
|
||||
mediaType: 'model/gltf-binary',
|
||||
},
|
||||
})
|
||||
|
||||
await uploadService.confirmFileUpload(userId, task.id, { fileId: asset.id })
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({ where: { id: asset.id } })
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const workflowTask = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: WorkflowTaskType.transcode_3d },
|
||||
})
|
||||
expect(workflowTask).toBeNull()
|
||||
})
|
||||
|
||||
it('should confirm file upload and resolve empty mediaType using Bun resolver for audio', async () => {
|
||||
const task = await prisma.task.create({
|
||||
data: { creatorId: userId, total: 1, uploaded: 0, type: 'upload' },
|
||||
|
||||
@@ -14,10 +14,11 @@ import {
|
||||
S3SignResponse,
|
||||
TaskInfo,
|
||||
} from '@shumai/dtos'
|
||||
import { ImageTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transcode'
|
||||
import { ImageTranscoder, ModelTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transcode'
|
||||
import { generateKeyBetween } from 'jittered-fractional-indexing'
|
||||
import { ulid } from 'ulid'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { getProxyType, isHtmlDocument, isOfficeDocument } from '@shumai/core/src/utils/mime'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
@@ -323,6 +324,7 @@ export class UploadService {
|
||||
const isImage = proxyType === 'image'
|
||||
const isAudio = proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
const is3D = proxyType === '3d'
|
||||
|
||||
if (!proxyType) {
|
||||
await tx.asset.update({
|
||||
@@ -361,7 +363,7 @@ export class UploadService {
|
||||
const isHtml = isHtmlDocument(asset.mediaType, asset.name)
|
||||
|
||||
if (isOffice || isHtml) {
|
||||
const gotenbergAvailable = await gotenbergService.isAvailable()
|
||||
const gotenbergAvailable = await gotenbergService.isAvailable(team.id)
|
||||
if (!gotenbergAvailable) {
|
||||
await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
@@ -376,6 +378,21 @@ export class UploadService {
|
||||
.withSprite()
|
||||
.withPoster()
|
||||
.submit()
|
||||
} else if (is3D) {
|
||||
const turntableAvailable = await turntableService.isAvailable(team.id)
|
||||
if (!turntableAvailable) {
|
||||
await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new ModelTranscoder(tx as any, asset.id, team.id, projectId)
|
||||
.withSprite()
|
||||
.withPoster()
|
||||
.submit()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,8 +126,26 @@ describe('getProxyType', () => {
|
||||
expect(getProxyType(null, 'slides.pptx')).toBe('pdf')
|
||||
})
|
||||
|
||||
it('should detect 3d proxyType for supported 3D models', () => {
|
||||
expect(getProxyType('model/gltf-binary', 'model.glb')).toBe('3d')
|
||||
expect(getProxyType(null, 'model.glb')).toBe('3d')
|
||||
expect(getProxyType(null, 'character.fbx')).toBe('3d')
|
||||
expect(getProxyType('model/obj', 'mesh.obj')).toBe('3d')
|
||||
expect(getProxyType(null, 'mesh.obj')).toBe('3d')
|
||||
expect(getProxyType('model/stl', 'part.stl')).toBe('3d')
|
||||
expect(getProxyType(null, 'part.stl')).toBe('3d')
|
||||
expect(getProxyType(null, 'scene.usd')).toBe('3d')
|
||||
expect(getProxyType(null, 'scene.usda')).toBe('3d')
|
||||
expect(getProxyType(null, 'scene.usdc')).toBe('3d')
|
||||
expect(getProxyType('model/vnd.usdz+zip', 'scene.usdz')).toBe('3d')
|
||||
expect(getProxyType('model/vnd.collada+xml', 'model.dae')).toBe('3d')
|
||||
expect(getProxyType(null, 'MODEL.GLB')).toBe('3d')
|
||||
})
|
||||
|
||||
it('should return null for unsupported files', () => {
|
||||
expect(getProxyType('application/zip', 'archive.zip')).toBeNull()
|
||||
expect(getProxyType(null, 'unknown.bin')).toBeNull()
|
||||
expect(getProxyType('model/gltf+json', 'model.gltf')).toBeNull()
|
||||
expect(getProxyType(null, 'model.gltf')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,43 @@
|
||||
import { isRawImage } from './raw'
|
||||
|
||||
export type ProxyType = 'image' | 'video' | 'audio' | 'pdf'
|
||||
export type ProxyType = 'image' | 'video' | 'audio' | 'pdf' | '3d'
|
||||
|
||||
export function is3dModel(mediaType?: string | null, filename?: string | null): boolean {
|
||||
const lowerMediaType = mediaType?.toLowerCase() || ''
|
||||
const lowerFilename = filename?.toLowerCase() || ''
|
||||
|
||||
if (lowerMediaType === 'model/gltf+json' || lowerFilename.endsWith('.gltf')) {
|
||||
return false
|
||||
}
|
||||
|
||||
const modelExtensions = [
|
||||
'.glb',
|
||||
'.usd',
|
||||
'.usda',
|
||||
'.usdc',
|
||||
'.usdz',
|
||||
'.fbx',
|
||||
'.obj',
|
||||
'.stl',
|
||||
'.dae',
|
||||
]
|
||||
if (modelExtensions.some((ext) => lowerFilename.endsWith(ext))) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (
|
||||
lowerMediaType.startsWith('model/') ||
|
||||
lowerMediaType === 'model/gltf-binary' ||
|
||||
lowerMediaType === 'model/obj' ||
|
||||
lowerMediaType === 'model/stl' ||
|
||||
lowerMediaType === 'model/vnd.usdz+zip' ||
|
||||
lowerMediaType === 'model/vnd.collada+xml'
|
||||
) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
export function isOfficeDocument(mediaType?: string | null, filename?: string | null): boolean {
|
||||
const lowerMediaType = mediaType?.toLowerCase() || ''
|
||||
@@ -97,6 +134,10 @@ export function getProxyType(
|
||||
return 'pdf'
|
||||
}
|
||||
|
||||
if (is3dModel(mediaType, filename)) {
|
||||
return '3d'
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
calculateTurntableFrameCenterTime,
|
||||
degreeToSecond,
|
||||
degreeToTurntableFrame,
|
||||
isValidTurntableDegree,
|
||||
secondToDegree,
|
||||
secondToTurntableFrame,
|
||||
TOTAL_TURNTABLE_FRAMES,
|
||||
TURNTABLE_FPS,
|
||||
} from './model3d'
|
||||
|
||||
describe('model3d utils', () => {
|
||||
describe('secondToTurntableFrame', () => {
|
||||
it('converts playback seconds to frame indices (0..23)', () => {
|
||||
expect(secondToTurntableFrame(0)).toBe(0)
|
||||
expect(secondToTurntableFrame(0.167)).toBe(1)
|
||||
expect(secondToTurntableFrame(0.5)).toBe(3)
|
||||
expect(secondToTurntableFrame(2.0)).toBe(12)
|
||||
expect(secondToTurntableFrame(3.9)).toBe(23)
|
||||
})
|
||||
|
||||
it('clamps out-of-range seconds', () => {
|
||||
expect(secondToTurntableFrame(-1)).toBe(0)
|
||||
expect(secondToTurntableFrame(10)).toBe(TOTAL_TURNTABLE_FRAMES - 1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('secondToDegree', () => {
|
||||
it('converts seconds to turntable degrees (0, 15, ..., 345)', () => {
|
||||
expect(secondToDegree(0)).toBe(0)
|
||||
expect(secondToDegree(1 / TURNTABLE_FPS)).toBe(15)
|
||||
expect(secondToDegree(0.5)).toBe(45)
|
||||
expect(secondToDegree(2.0)).toBe(180)
|
||||
expect(secondToDegree(3.9)).toBe(345)
|
||||
})
|
||||
})
|
||||
|
||||
describe('degreeToTurntableFrame', () => {
|
||||
it('converts discrete degrees to frame indices', () => {
|
||||
expect(degreeToTurntableFrame(0)).toBe(0)
|
||||
expect(degreeToTurntableFrame(15)).toBe(1)
|
||||
expect(degreeToTurntableFrame(45)).toBe(3)
|
||||
expect(degreeToTurntableFrame(90)).toBe(6)
|
||||
expect(degreeToTurntableFrame(180)).toBe(12)
|
||||
expect(degreeToTurntableFrame(345)).toBe(23)
|
||||
})
|
||||
|
||||
it('normalizes degrees outside [0, 360)', () => {
|
||||
expect(degreeToTurntableFrame(360)).toBe(0)
|
||||
expect(degreeToTurntableFrame(-15)).toBe(23)
|
||||
expect(degreeToTurntableFrame(720)).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('calculateTurntableFrameCenterTime and degreeToSecond', () => {
|
||||
it('calculates frame center times within bounds', () => {
|
||||
const centerTime0 = calculateTurntableFrameCenterTime(0)
|
||||
expect(centerTime0).toBeCloseTo(0.5 / TURNTABLE_FPS)
|
||||
|
||||
const centerTime23 = calculateTurntableFrameCenterTime(23)
|
||||
expect(centerTime23).toBeLessThan(TOTAL_TURNTABLE_FRAMES / TURNTABLE_FPS)
|
||||
expect(centerTime23).toBeGreaterThan(23 / TURNTABLE_FPS)
|
||||
|
||||
expect(degreeToSecond(0)).toBeCloseTo(centerTime0)
|
||||
expect(degreeToSecond(45)).toBeCloseTo((3 + 0.5) / TURNTABLE_FPS)
|
||||
expect(degreeToSecond(345)).toBeCloseTo(centerTime23)
|
||||
})
|
||||
})
|
||||
|
||||
describe('isValidTurntableDegree', () => {
|
||||
it('returns true for integer multiples of 15 from 0 to 345', () => {
|
||||
for (let deg = 0; deg < 360; deg += 15) {
|
||||
expect(isValidTurntableDegree(deg)).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('returns false for non-multiples of 15', () => {
|
||||
expect(isValidTurntableDegree(10)).toBe(false)
|
||||
expect(isValidTurntableDegree(44)).toBe(false)
|
||||
expect(isValidTurntableDegree(45.5)).toBe(false)
|
||||
})
|
||||
|
||||
it('returns false for out of range or negative degrees', () => {
|
||||
expect(isValidTurntableDegree(-15)).toBe(false)
|
||||
expect(isValidTurntableDegree(360)).toBe(false)
|
||||
expect(isValidTurntableDegree(720)).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,58 @@
|
||||
export const TOTAL_TURNTABLE_FRAMES = 24
|
||||
export const TURNTABLE_FPS = 6
|
||||
export const TURNTABLE_FRAME_DURATION = 1 / TURNTABLE_FPS
|
||||
|
||||
/**
|
||||
* Converts a playback second into a turntable frame index (0..23), clamped.
|
||||
*/
|
||||
export function secondToTurntableFrame(second: number): number {
|
||||
return Math.min(
|
||||
Math.max(0, Math.floor(second * TURNTABLE_FPS + 1e-4)),
|
||||
TOTAL_TURNTABLE_FRAMES - 1,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a turntable playback second to turntable degree (0, 15, 30, ..., 345).
|
||||
*/
|
||||
export function secondToDegree(second: number): number {
|
||||
const frameIndex = secondToTurntableFrame(second)
|
||||
return (frameIndex * 15) % 360
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the presentation time within a turntable frame.
|
||||
* Seeking near frame center avoids PTS boundary inaccuracies in video decoders.
|
||||
*/
|
||||
export function calculateTurntableFrameCenterTime(frameIndex: number): number {
|
||||
const normalizedFrame =
|
||||
((frameIndex % TOTAL_TURNTABLE_FRAMES) + TOTAL_TURNTABLE_FRAMES) % TOTAL_TURNTABLE_FRAMES
|
||||
if (normalizedFrame === TOTAL_TURNTABLE_FRAMES - 1) {
|
||||
return normalizedFrame * TURNTABLE_FRAME_DURATION + TURNTABLE_FRAME_DURATION * 0.25
|
||||
}
|
||||
return (normalizedFrame + 0.5) * TURNTABLE_FRAME_DURATION
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a rotation degree (0..345) to a turntable frame index (0..23).
|
||||
*/
|
||||
export function degreeToTurntableFrame(degree: number): number {
|
||||
const normalizedDegree = ((Math.round(degree) % 360) + 360) % 360
|
||||
return Math.round(normalizedDegree / 15) % TOTAL_TURNTABLE_FRAMES
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a rotation degree into a turntable video playback second (center time of that frame).
|
||||
*/
|
||||
export function degreeToSecond(degree: number): number {
|
||||
const frameIndex = degreeToTurntableFrame(degree)
|
||||
return calculateTurntableFrameCenterTime(frameIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates whether a degree is a valid discrete turntable angle:
|
||||
* integer multiple of 15 between 0 and 345 inclusive (0, 15, 30, ..., 345).
|
||||
*/
|
||||
export function isValidTurntableDegree(degree: number): boolean {
|
||||
return Number.isInteger(degree) && degree >= 0 && degree <= 345 && degree % 15 === 0
|
||||
}
|
||||
@@ -258,6 +258,57 @@ describe('WatermarkService', () => {
|
||||
expect(mediaIds).toContain(mediaFile.id)
|
||||
})
|
||||
|
||||
it('should include 3D assets with turntable video in getMediaAssetIdsInShareLink', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: '3D User', email: '3d-wm@example.com' },
|
||||
})
|
||||
const team = await teamService.createTeam(user, { name: '3D Team' })
|
||||
|
||||
const projectFolder = await prisma.asset.create({
|
||||
data: { name: 'root', type: 'root', status: 'processed' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: '3D Project', teamId: team.id, rootFolderId: projectFolder.id },
|
||||
})
|
||||
|
||||
const shareLink = await shareService.createShareLink(project.id, { name: '3D Share' }, user.id)
|
||||
|
||||
const modelAsset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: 'file',
|
||||
mediaType: 'model/gltf-binary',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
media: {
|
||||
duration: 4,
|
||||
filesize: 100,
|
||||
frames: 24,
|
||||
proxyType: '3d',
|
||||
videoTranscodes: [
|
||||
{ key: 'files/asset-3d/turntable.mp4', width: 1080, height: 1080, resolution: '1080p' },
|
||||
],
|
||||
finishedAt: new Date().toISOString(),
|
||||
original: { key: 'files/asset-3d/robot.glb', filesizeInBytes: 100, codec: '' },
|
||||
} as PrismaJson.MediaInfo,
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'robot.glb',
|
||||
type: 'symlink',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
parentId: shareLink.rootFolderId,
|
||||
targetId: modelAsset.id,
|
||||
},
|
||||
})
|
||||
|
||||
const mediaIds = await watermarkService.getMediaAssetIdsInShareLink(shareLink.rootFolderId)
|
||||
expect(mediaIds).toContain(modelAsset.id)
|
||||
})
|
||||
|
||||
describe('purgeOrphanWatermarkConfigs', () => {
|
||||
const bucket = 'shumai-gc-test-bucket'
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
} from '@shumai/core/src/share/errors'
|
||||
import { shareService } from '@shumai/core/src/share/share'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { is3dModel } from '@shumai/core/src/utils/mime'
|
||||
|
||||
// Union of the global client and interactive-transaction client, which share the
|
||||
// same delegate shapes for the models used by this service.
|
||||
@@ -319,8 +320,9 @@ export class WatermarkService {
|
||||
const proxyType = mediaInfo?.proxyType
|
||||
const isVideo = proxyType === 'video' || asset.mediaType?.startsWith('video/')
|
||||
const isImage = proxyType === 'image' || asset.mediaType?.startsWith('image/')
|
||||
const is3d = proxyType === '3d' || is3dModel(asset.mediaType)
|
||||
|
||||
if (isVideo || isImage) {
|
||||
if (isVideo || isImage || is3d) {
|
||||
mediaAssetIds.push(asset.id)
|
||||
}
|
||||
} else if (asset.type === 'folder' || asset.type === 'version_stack') {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/db",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
-- AlterEnum
|
||||
ALTER TYPE "WorkflowTaskType" ADD VALUE 'transcode_3d';
|
||||
@@ -0,0 +1,8 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "asset_embeddings" ADD COLUMN "embedding_768" vector(768);
|
||||
|
||||
-- CreateIndex: HNSW index on asset_embeddings.embedding_768
|
||||
CREATE INDEX "AssetEmbedding_embedding_768_idx"
|
||||
ON "asset_embeddings"
|
||||
USING hnsw (embedding_768 vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 64);
|
||||
@@ -121,6 +121,7 @@ enum WorkflowTaskType {
|
||||
transcode_screenshot
|
||||
transcode_image_annotation
|
||||
transcode_watermark
|
||||
transcode_3d
|
||||
ai_metadata_autofill
|
||||
chat
|
||||
ai_embedding
|
||||
@@ -705,17 +706,19 @@ model AssetCommentReaction {
|
||||
}
|
||||
|
||||
model AssetEmbedding {
|
||||
id String @id @default(ulid())
|
||||
embedding Unsupported("vector(1536)")?
|
||||
startTime Float? @map("start_time")
|
||||
endTime Float? @map("end_time")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
id String @id @default(ulid())
|
||||
embedding Unsupported("vector(1536)")?
|
||||
embedding768 Unsupported("vector(768)")? @map("embedding_768")
|
||||
startTime Float? @map("start_time")
|
||||
endTime Float? @map("end_time")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
assetId String @map("asset_id")
|
||||
asset Asset @relation("AssetToEmbedding", fields: [assetId], references: [id], onDelete: Cascade)
|
||||
assetId String @map("asset_id")
|
||||
asset Asset @relation("AssetToEmbedding", fields: [assetId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([embedding], name: "AssetEmbedding_embedding_idx")
|
||||
@@index([embedding768], name: "AssetEmbedding_embedding_768_idx")
|
||||
@@map("asset_embeddings")
|
||||
}
|
||||
|
||||
|
||||
+1
@@ -150,6 +150,7 @@ export const WorkflowTaskType = {
|
||||
transcode_screenshot: 'transcode_screenshot',
|
||||
transcode_image_annotation: 'transcode_image_annotation',
|
||||
transcode_watermark: 'transcode_watermark',
|
||||
transcode_3d: 'transcode_3d',
|
||||
ai_metadata_autofill: 'ai_metadata_autofill',
|
||||
chat: 'chat',
|
||||
ai_embedding: 'ai_embedding',
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -103,8 +103,22 @@ declare global {
|
||||
replyTo?: string
|
||||
}
|
||||
|
||||
export interface TurntableSettings {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
}
|
||||
|
||||
export interface GotenbergSettings {
|
||||
url?: string
|
||||
username?: string
|
||||
password?: string
|
||||
}
|
||||
|
||||
export interface Settings {
|
||||
transcode?: TranscodeSettings
|
||||
turntable?: TurntableSettings
|
||||
gotenberg?: GotenbergSettings
|
||||
mediaGeneration?: MediaGenerationSettings
|
||||
appearance?: AppearanceSettings
|
||||
emailNotification?: EmailNotificationSettings
|
||||
@@ -196,7 +210,7 @@ declare global {
|
||||
duration: number
|
||||
filesize: number
|
||||
frames: number
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf'
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | '3d'
|
||||
imageTranscodes: ImageTranscode[]
|
||||
videoTranscodes: VideoTranscode[]
|
||||
videoPreview?: VideoTranscode
|
||||
@@ -530,7 +544,7 @@ declare global {
|
||||
key: string
|
||||
sizeByte: number
|
||||
contentType?: string | null
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | '3d' | null
|
||||
}
|
||||
export type KanbanCommentAttachmentList = KanbanCommentAttachment[]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -42,6 +42,7 @@ export const agentInfoSchema = z.object({
|
||||
skills: z.array(agentSkillSchema).optional(),
|
||||
mcpServerIds: z.array(z.string()).optional(),
|
||||
deniedTools: z.array(z.string()).optional(),
|
||||
readOnly: z.boolean().optional(),
|
||||
})
|
||||
export type AgentInfo = z.infer<typeof agentInfoSchema>
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ export const assetInfoPaginatedListSchema = z.object({
|
||||
export type AssetInfoPaginatedList = z.infer<typeof assetInfoPaginatedListSchema>
|
||||
|
||||
export const previewInfoSchema = z.object({
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
thumbnailUrl: z.string().optional(),
|
||||
originalHeight: z.number().optional(),
|
||||
originalWidth: z.number().optional(),
|
||||
@@ -76,7 +76,7 @@ export const assetInfoSchema = z.object({
|
||||
type: z.string(),
|
||||
targetType: z.string().optional().nullable(),
|
||||
status: z.string(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
latestChildren: z.array(childPreviewSchema).optional(),
|
||||
preview: previewInfoSchema.nullable().optional(),
|
||||
createdAt: z.string(),
|
||||
@@ -143,7 +143,7 @@ export const assetInfoSchema = z.object({
|
||||
.optional(),
|
||||
})
|
||||
.optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
metadata: mediaMetadataSchema.optional(),
|
||||
error: z.string().optional(),
|
||||
})
|
||||
@@ -233,7 +233,7 @@ export const attachmentInfoSchema = z.object({
|
||||
id: z.string(),
|
||||
assetId: z.string(),
|
||||
url: z.string(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
})
|
||||
export type AttachmentInfo = z.infer<typeof attachmentInfoSchema>
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ export const shumaiAssetContextSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
type: z.string(),
|
||||
mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(),
|
||||
mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(),
|
||||
mimeType: z.string().optional(),
|
||||
parentId: z.string().optional(),
|
||||
path: z.string().optional(),
|
||||
@@ -64,7 +64,7 @@ export const shumaiAttachedFileContextSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
type: z.string(),
|
||||
mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(),
|
||||
mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(),
|
||||
mimeType: z.string().optional(),
|
||||
path: z.string().optional(),
|
||||
url: z.string().optional(),
|
||||
@@ -75,6 +75,7 @@ export type ShumaiAttachedFileContext = z.infer<typeof shumaiAttachedFileContext
|
||||
export const shumaiMediaPositionSchema = z.discriminatedUnion('type', [
|
||||
z.object({ type: z.literal('time'), seconds: z.number() }),
|
||||
z.object({ type: z.literal('page'), page: z.number() }),
|
||||
z.object({ type: z.literal('degree'), degree: z.number().int() }),
|
||||
])
|
||||
|
||||
export type ShumaiMediaPosition = z.infer<typeof shumaiMediaPositionSchema>
|
||||
|
||||
@@ -104,6 +104,8 @@ export function serializeContextToXml(context?: ShumaiMessageContext | null): st
|
||||
lines.push(` <position type="time" seconds="${context.position.seconds.toFixed(4)}" />`)
|
||||
} else if (context.position.type === 'page') {
|
||||
lines.push(` <position type="page" page="${Math.round(context.position.page)}" />`)
|
||||
} else if (context.position.type === 'degree') {
|
||||
lines.push(` <position type="degree" degree="${Math.round(context.position.degree)}" />`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ export const kanbanAttachmentPayloadSchema = z.object({
|
||||
key: z.string(),
|
||||
sizeByte: z.number(),
|
||||
contentType: z.string().nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
})
|
||||
export type KanbanAttachmentPayload = z.infer<typeof kanbanAttachmentPayloadSchema>
|
||||
|
||||
@@ -17,7 +17,7 @@ export const kanbanAttachmentInfoSchema = z.object({
|
||||
sizeByte: z.number(),
|
||||
contentType: z.string().nullable().optional(),
|
||||
url: z.string(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
})
|
||||
export type KanbanAttachmentInfo = z.infer<typeof kanbanAttachmentInfoSchema>
|
||||
|
||||
@@ -28,7 +28,7 @@ export const postKanbanAttachmentResponseSchema = z.object({
|
||||
sizeByte: z.number(),
|
||||
contentType: z.string().nullable().optional(),
|
||||
uploadUrl: z.string(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf', '3d']).nullable().optional(),
|
||||
})
|
||||
export type PostKanbanAttachmentResponse = z.infer<typeof postKanbanAttachmentResponseSchema>
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user