Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2cbc86fdd6 | ||
|
|
d391705519 | ||
|
|
34a69a33bf | ||
|
|
d32707d038 | ||
|
|
89777e0b6d | ||
|
|
8b800bc792 | ||
|
|
899b600417 | ||
|
|
954d8ee497 | ||
|
|
86ad68a705 | ||
|
|
d2ee7f1c45 | ||
|
|
7cb1aa3c86 | ||
|
|
1704fddc67 | ||
|
|
07d82377bd | ||
|
|
9c8f29e3ba | ||
|
|
9fd707b23f | ||
|
|
14f6752de2 | ||
|
|
b75b276908 | ||
|
|
95f91cd4de | ||
|
|
bb65c5a0d3 | ||
|
|
a585718660 | ||
|
|
adb5f58535 | ||
|
|
9ca9b2d8e4 | ||
|
|
b173b4910e | ||
|
|
9576453a72 | ||
|
|
5233615720 | ||
|
|
1ebf79edd3 | ||
|
|
eee7b5a677 | ||
|
|
837fe5bd7a | ||
|
|
f159fec693 | ||
|
|
dc46853582 | ||
|
|
850b1ff667 | ||
|
|
4475c7bed6 | ||
|
|
0fd826b973 | ||
|
|
2d5b4a3753 | ||
|
|
fb63d8b7b6 | ||
|
|
7182c00c7d | ||
|
|
e8058a2292 | ||
|
|
1b60c8df83 | ||
|
|
4dc53063ea | ||
|
|
d5dbb3c01b | ||
|
|
8779ae52a2 | ||
|
|
5c21463075 | ||
|
|
914855a25c | ||
|
|
c7b0346782 | ||
|
|
e6818860da | ||
|
|
5dae5740e3 | ||
|
|
bbe36a028d | ||
|
|
ba5f7296a8 | ||
|
|
cd4a22d20b | ||
|
|
fc7d1452b0 | ||
|
|
4fd27ab147 | ||
|
|
6e2e5bdbc2 | ||
|
|
b6ff60f8b8 | ||
|
|
48f4c83cc8 | ||
|
|
81b0083672 | ||
|
|
3737418ec3 | ||
|
|
6d47ea7466 | ||
|
|
84d96ff83a | ||
|
|
6e985aed61 | ||
|
|
8347a9ca15 | ||
|
|
ab9b33320d | ||
|
|
efd6163238 | ||
|
|
0e4bdec0e3 | ||
|
|
0eb03b16a4 | ||
|
|
b216773044 | ||
|
|
92801ba14a | ||
|
|
661e836f06 | ||
|
|
fb5c7ecabe | ||
|
|
a390643458 | ||
|
|
9325c71b51 | ||
|
|
2f746d3ce8 | ||
|
|
bdceda4cb6 | ||
|
|
df6e392bf6 | ||
|
|
48b2047a84 | ||
|
|
2b360b2b2c | ||
|
|
5d19e6ddfa | ||
|
|
5e70b6f3ee | ||
|
|
8eab02ab28 | ||
|
|
7aa347df92 | ||
|
|
5d6462adf0 | ||
|
|
5f1e0c1823 | ||
|
|
d8381dff21 | ||
|
|
b6519431db | ||
|
|
02fddf8ebd | ||
|
|
f4c52d9273 | ||
|
|
77093aa294 | ||
|
|
4806c63906 | ||
|
|
14d04e9d77 | ||
|
|
342cdcdd06 | ||
|
|
98a76e913b | ||
|
|
2240901d55 | ||
|
|
6892967993 | ||
|
|
7ea5345c52 | ||
|
|
bea81a207a | ||
|
|
c20ef0e4b6 | ||
|
|
754131f13a | ||
|
|
6981706e13 | ||
|
|
280c06b68c | ||
|
|
4e21becbe0 | ||
|
|
86c1d964a3 | ||
|
|
87fc10b62f | ||
|
|
7d91d70a14 | ||
|
|
1c994adce9 | ||
|
|
afd55eca85 | ||
|
|
29e3d3ae03 | ||
|
|
90ef063d43 | ||
|
|
e33cbdb37c | ||
|
|
0514a19702 | ||
|
|
9fb9e608b9 | ||
|
|
cbb25b49e7 | ||
|
|
ce8fc5d128 | ||
|
|
34bacf5421 | ||
|
|
98bbfa51ae | ||
|
|
fc9dd8a5f7 | ||
|
|
c7fef79860 | ||
|
|
efa0e75a06 | ||
|
|
b5ff76960f | ||
|
|
ca0c2ece75 | ||
|
|
c8b1d521d8 | ||
|
|
4bd5083ac1 | ||
|
|
c6527650d0 | ||
|
|
d4a1eff715 | ||
|
|
463cd61cfd | ||
|
|
93b4656939 | ||
|
|
3e122a86f0 | ||
|
|
292307e28e | ||
|
|
982bca9383 | ||
|
|
d1061a1023 | ||
|
|
994f1d3b44 | ||
|
|
e057f17f8e | ||
|
|
66d07d1fb0 | ||
|
|
5542538a64 | ||
|
|
7753721731 | ||
|
|
ef86cc2713 | ||
|
|
cc79be7e34 | ||
|
|
a9c5a3d9fd | ||
|
|
0d9545f0b5 | ||
|
|
2925fb6bf6 | ||
|
|
7ae34a4bc0 | ||
|
|
f645c8fefa | ||
|
|
1835016a9b | ||
|
|
5e92660d1c | ||
|
|
6cb3ca6874 | ||
|
|
5d14f4fff1 | ||
|
|
009566d42f | ||
|
|
50777d332b | ||
|
|
852a9873dc | ||
|
|
aa265c5322 | ||
|
|
01ab254c21 | ||
|
|
ca5ca5e080 | ||
|
|
3f27439733 | ||
|
|
2a611f9b5a | ||
|
|
7137e00911 | ||
|
|
59db97aedf | ||
|
|
a76dbdfc22 | ||
|
|
61e2a3c53a | ||
|
|
7707498916 | ||
|
|
91c8af86db | ||
|
|
286d71133f | ||
|
|
53c132e41d | ||
|
|
5c285b172b | ||
|
|
f1d52ac44e | ||
|
|
ec6736b452 | ||
|
|
46c975836a | ||
|
|
5d7ea7d118 | ||
|
|
5065346e8a | ||
|
|
60d469c91b | ||
|
|
f68c1c56e1 | ||
|
|
08cf90477a | ||
|
|
e01c348fde | ||
|
|
b0eb5b1eef | ||
|
|
3b7fc410b2 | ||
|
|
4c7ff6f58f | ||
|
|
a197c83580 | ||
|
|
718d9d5b51 | ||
|
|
212ac3bc0e | ||
|
|
eaf2c867b3 | ||
|
|
570d95fa6f | ||
|
|
32892a21c1 | ||
|
|
7819a2034e | ||
|
|
e0f2698a66 | ||
|
|
72c50ffd73 | ||
|
|
d84950d372 | ||
|
|
46815736e2 | ||
|
|
9d1e8378f7 | ||
|
|
f399d2c469 | ||
|
|
4af081f128 | ||
|
|
ce457e1161 | ||
|
|
a69dd6a660 | ||
|
|
62d7061cb3 | ||
|
|
1ad5720f42 | ||
|
|
0be74dd4ab | ||
|
|
f4276a19aa | ||
|
|
d81fc275f4 | ||
|
|
ead91455d8 | ||
|
|
438ea79535 | ||
|
|
3455cd0b13 | ||
|
|
2aba1c898b | ||
|
|
6f91b9e809 | ||
|
|
10760f5a57 | ||
|
|
c150f62b4c | ||
|
|
1284c1de3f | ||
|
|
0c89616a4e | ||
|
|
96a682b414 | ||
|
|
5f8067cbe8 | ||
|
|
ae39a98e84 | ||
|
|
0944f1ad8e | ||
|
|
812a8b3b88 | ||
|
|
24a2f74035 | ||
|
|
73d02bf319 | ||
|
|
6e98d6ee4a | ||
|
|
af381ff99c | ||
|
|
c9da7794d9 | ||
|
|
1fb03cd852 | ||
|
|
74fff87a2b | ||
|
|
3af5940ddf | ||
|
|
3bb44041d7 | ||
|
|
29b7124f48 | ||
|
|
718f50ef10 | ||
|
|
006774042a | ||
|
|
2b058bde40 | ||
|
|
fc6ad20329 | ||
|
|
7e5735f0cc | ||
|
|
6e74ede9fe | ||
|
|
b8d876d28e | ||
|
|
95984f19b1 | ||
|
|
5111fcd6a5 | ||
|
|
f16a8a7069 | ||
|
|
7f6e41708a | ||
|
|
6fa69d3310 | ||
|
|
7c5f462ba5 | ||
|
|
5d778f1b43 | ||
|
|
6027c2c21c | ||
|
|
d0931b7181 | ||
|
|
2220d35e40 | ||
|
|
bbf55e22da | ||
|
|
68d0830d05 | ||
|
|
2858274e52 | ||
|
|
45318d1eec | ||
|
|
e8e04edfee | ||
|
|
cf2546c861 | ||
|
|
1f640ede19 | ||
|
|
3b4751bcb2 | ||
|
|
d766edbbd0 | ||
|
|
673041e28a | ||
|
|
eabf49f6e3 | ||
|
|
f226bc74c6 | ||
|
|
2159ddfeb5 | ||
|
|
813c003a10 | ||
|
|
720710dbbf | ||
|
|
68a1714e7f | ||
|
|
d8aaed5212 | ||
|
|
60c499c153 | ||
|
|
a9c6686843 | ||
|
|
c2f6c85194 | ||
|
|
8e012f9353 | ||
|
|
c55abbf724 | ||
|
|
e9e096ac9e | ||
|
|
6a63e99044 | ||
|
|
9c3b808809 | ||
|
|
c6387dd4ef | ||
|
|
a1dfc8b4f9 | ||
|
|
443b099e12 | ||
|
|
e9e9be71f4 | ||
|
|
3ae74d6417 | ||
|
|
821de71719 | ||
|
|
cbc0a7960d | ||
|
|
0e2384627d | ||
|
|
0a0a8f973d | ||
|
|
fc3baede99 | ||
|
|
ca31ef02fd | ||
|
|
2824377600 | ||
|
|
f285b9e327 | ||
|
|
42d89eb88d | ||
|
|
8d8eeaaf47 | ||
|
|
8c55897d80 | ||
|
|
1790149986 | ||
|
|
88e5440d5c | ||
|
|
d687ea495b | ||
|
|
e437cdfeaf | ||
|
|
0294056ad9 | ||
|
|
5cf949abc0 | ||
|
|
e21d419bfe | ||
|
|
4159c0737c | ||
|
|
9a2bf1deaf | ||
|
|
1a43fad77b | ||
|
|
b5fb378cf8 | ||
|
|
782316ca97 | ||
|
|
e5084cc8b3 | ||
|
|
02b009fbe1 | ||
|
|
6dcd08ae3a | ||
|
|
d230a1fd4a | ||
|
|
210ed5f95a | ||
|
|
edd550b9f2 | ||
|
|
ea71954deb | ||
|
|
dc63ab3eaf | ||
|
|
d06dfc0ab3 | ||
|
|
84aa1662be | ||
|
|
91fedf1477 | ||
|
|
b0dbb56493 | ||
|
|
f6a0577585 | ||
|
|
1fee0f5a97 |
@@ -1,4 +1,6 @@
|
||||
node_modules
|
||||
dist
|
||||
data
|
||||
Dockerfile*
|
||||
docker-compose*
|
||||
.dockerignore
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
src/generated/** linguist-generated
|
||||
packages/db/src/generated/** linguist-generated
|
||||
packages/webui/paraglide/** linguist-generated
|
||||
|
||||
@@ -46,6 +46,12 @@ jobs:
|
||||
- name: Test Local Release
|
||||
run: |
|
||||
bun run release:local --skip-check --out /tmp/shumai-local-release --force
|
||||
cd /tmp/shumai-local-release/node
|
||||
npx shumai-agent
|
||||
npx shumai-transcode
|
||||
|
||||
echo "Smoke testing shumai..."
|
||||
bun run /tmp/shumai-local-release/node/node_modules/@shumai-one/shumai/bin/shumai-app.js --check
|
||||
|
||||
echo "Smoke testing shumai-agent..."
|
||||
bun run /tmp/shumai-local-release/node/node_modules/@shumai-one/shumai-agent/bin/shumai-agent-app.js --check
|
||||
|
||||
echo "Smoke testing shumai-transcode..."
|
||||
bun run /tmp/shumai-local-release/node/node_modules/@shumai-one/shumai-transcode/bin/shumai-transcode-app.js --check
|
||||
|
||||
@@ -2,4 +2,8 @@ node_modules
|
||||
dist
|
||||
.bun
|
||||
src/generated
|
||||
src/ui/components/ui
|
||||
packages/db/src/generated
|
||||
src/ui/components/ui
|
||||
packages/webui/paraglide
|
||||
*.md
|
||||
**/*.md
|
||||
@@ -244,7 +244,7 @@ The `bunfig.toml` is configured to run `setup-tests.ts` automatically as a prelo
|
||||
|
||||
### API Tests
|
||||
|
||||
- Located in `packages/api/src/api/**/*.test.ts`.
|
||||
- Located in `packages/api/src/**/*.test.ts`.
|
||||
- Mock the Service layer using `vi.spyOn(service, 'method')` or `vi.mock('@shumai/core/src/team/team')`.
|
||||
- Verify that the API calls the Service methods correctly.
|
||||
|
||||
@@ -396,6 +396,7 @@ To prevent Temporal from indefinitely retrying fatal, expected business validati
|
||||
## Radix UI / Shadcn UI Patterns
|
||||
|
||||
- **Modal Overlays**: When triggering a dialog (e.g., `Dialog`, `AlertDialog`) from inside a `DropdownMenu`, you must set `modal={false}` on the `<DropdownMenu>` component. Failing to do so will cause the UI to freeze due to conflicting focus management between the two modal components.
|
||||
- **Scrollable Content**: Always use the Shadcn `ScrollArea` component (`@/ui/components/ui/scroll-area`) for scrollable content containers to ensure consistent custom scrollbar UI across the application.
|
||||
|
||||
## Logging
|
||||
|
||||
@@ -407,3 +408,51 @@ We use **pino** for logging. Always use **structured logging** to ensure logs ar
|
||||
logger.info({ userId: user.id, projectId }, 'Project deleted successfully')
|
||||
```
|
||||
- **Levels**: Use appropriate levels: `debug`, `info`, `warn`, `error`.
|
||||
|
||||
## Internationalization (i18n)
|
||||
|
||||
All user-facing text in the WebUI (`packages/webui`) MUST be internationalized using **Paraglide JS**. Hardcoded text in UI components is strictly forbidden.
|
||||
|
||||
### Message Definition
|
||||
|
||||
1. Define your English message strings in [packages/webui/messages/en.json](packages/webui/messages/en.json).
|
||||
2. Define the corresponding Chinese translation strings in [packages/webui/messages/zh.json](packages/webui/messages/zh.json).
|
||||
3. Keys should use `snake_case`.
|
||||
|
||||
Example (`en.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"my_key": "My English Text"
|
||||
}
|
||||
```
|
||||
|
||||
### Compilation
|
||||
|
||||
When you add or update translation keys, you must compile them to regenerate the type-safe functions:
|
||||
|
||||
```bash
|
||||
# Compile once (generates type-safe TS/JS functions on disk)
|
||||
bun run i18n:compile
|
||||
|
||||
# Watch and recompile on change during development
|
||||
bun run i18n:watch
|
||||
```
|
||||
|
||||
### Usage in Components
|
||||
|
||||
Import the compiled message namespace `m` and call it as a function:
|
||||
|
||||
```typescript
|
||||
import { m } from '@/ui/paraglide/messages.js'
|
||||
|
||||
export function MyComponent() {
|
||||
return (
|
||||
<div>
|
||||
<h1>{m.my_key()}</h1>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
Always use the absolute workspace alias path `@/ui/paraglide/messages.js` (or `runtime.js` for runtime features) when importing Paraglide outputs.
|
||||
|
||||
@@ -47,6 +47,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
socat \
|
||||
ca-certificates \
|
||||
bubblewrap \
|
||||
gosu \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -69,5 +70,12 @@ COPY --chown=bun:bun --from=builder /app/prisma.config.ts.prod ./prisma.config.t
|
||||
# Expose port (Bun Hono defaults to 3000)
|
||||
EXPOSE 3000
|
||||
|
||||
# Switch back to root at the end so the container runs the entrypoint as root
|
||||
USER root
|
||||
COPY scripts/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
|
||||
# Default command: run migrations and start API server
|
||||
CMD ["sh", "-c", "bun prisma migrate deploy && bun apps/web/src/index.js"]
|
||||
CMD ["sh", "-c", "bun prisma migrate deploy && bun index.js"]
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
<p align="center">
|
||||
<img src="docs/logo-rounded.png" alt="Shumai Logo" width="65" />
|
||||
</p>
|
||||
|
||||
<h1 align="center">Shumai</h1>
|
||||
|
||||
<p align="center">
|
||||
<strong>An open-source platform for all your creative work.</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a target="_blank" href="https://shumai.one/">Website</a> | <a target="_blank" href="https://docs.shumai.one/introduction">Docs</a> | <a target="_blank" href="https://staging.shumai.one">Demo</a>
|
||||
</p>
|
||||
|
||||

|
||||
|
||||
## Features
|
||||
|
||||
- **S3-Compatible & Local Storage**: Securely store and serve your creative assets using a local filesystem or any S3-compatible cloud storage (AWS S3, Cloudflare R2, MinIO, etc.).
|
||||
- **Frame-by-Frame Annotations & Comments**: Give precise feedback using frame-specific drawing tools and timestamped comments directly on video and image assets.
|
||||
- **Secure Sharing & Collections**: Create secure public share links and curated media collections to collaborate with clients and stakeholders.
|
||||
- **Granular Access Control**: Manage workspace permissions using team-level and project-level role-based access controls.
|
||||
- **Distributed Transcoding via Temporal**: Offload resource-heavy video transcoding to a background worker pool orchestrated by Temporal.
|
||||
- **Custom Asset Metadata**: Define and customize dynamic metadata fields tailored to your production pipeline.
|
||||
|
||||
#### Shumai Agent
|
||||
|
||||
- **Collaborative AI Chat**: Converse with a context-aware AI agent directly within your project workspace.
|
||||
- **Custom Skills & Tools**: Extend the agent's capabilities by registering custom scripts, tools, and automation skills.
|
||||
- **Isolated Sandbox Execution**: Run agent-submitted scripts securely within a sandboxed environment.
|
||||
- **AI-Powered Metadata Autofill**: Automatically generate tags, descriptions, and custom metadata for new assets using Google Gemini.
|
||||
- **Semantic Search**: Locate assets instantly based on visual or conceptual search queries using vector embeddings.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
Below is a quickstart guide for running Shumai with local storage. For advanced configuration options (including S3-compatible storage and Temporal workflow orchestration), see our [Documentation](https://docs.shumai.one/getting-started/overview).
|
||||
|
||||
### Option 1: Docker Compose
|
||||
|
||||
Docker Compose is the fastest way to get Shumai running. You do not need to clone the repository or install packages manually. Ensure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed, then follow these steps:
|
||||
|
||||
1. Create and navigate to a new directory for your configuration and data volumes:
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
2. Download the `docker-compose.yaml` file:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.yaml
|
||||
```
|
||||
3. Configure environment variables (optional):
|
||||
|
||||
* `SHUMAI_SERVER_PORT` controls the port the Shumai server listens on. The default is `3000`.
|
||||
* By default, Shumai uses the bundled local storage service in this Docker Compose setup. To use an external S3-compatible storage instead, set `AWS_ENDPOINT_URL_S3` to the provider's endpoint URL and configure the corresponding AWS credentials and bucket settings as needed.
|
||||
```
|
||||
AWS_ENDPOINT_URL_S3: S3_ENDPOINT_URL
|
||||
S3_REGION: S3_REGION
|
||||
S3_BUCKET: BUCKET_NAME
|
||||
S3_ACCESS_KEY_ID: ID
|
||||
S3_SECRET_ACCESS_KEY: KEY
|
||||
```
|
||||
* By default, `AWS_ENDPOINT_URL_S3` is set to: `http://localhost:{SHUMAI_SERVER_PORT}`, if you use local storage and expose Shumai on a custom host/port combination, set `AWS_ENDPOINT_URL_S3` to the external URL that browsers will use to upload files.
|
||||
|
||||
For example, if you change the port mapping in `docker-compose.yaml` from `3000:3000` to `12345:3000` and deploy on a server with IP address `12.34.56.78`, set:
|
||||
|
||||
```
|
||||
AWS_ENDPOINT_URL_S3: http://12.34.56.78:12345
|
||||
```
|
||||
|
||||
This value must be reachable from client browsers and should include the externally exposed port.
|
||||
|
||||
5. Start the services in detached mode:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
6. Open your browser and access Shumai at `http://localhost:3000` (or `http://<your-server-ip>:3000` for remote deployments).
|
||||
|
||||
### Option 2: Install via NPM / Package Manager
|
||||
|
||||
Shumai is published as `@shumai-one/shumai` on NPM. This option allows you to run Shumai globally or locally.
|
||||
|
||||
#### Step 1: Start PostgreSQL with pgvector
|
||||
|
||||
Shumai requires PostgreSQL with the `pgvector` extension. Start a pre-configured database container using Docker:
|
||||
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
-e POSTGRES_USER=shumai \
|
||||
-e POSTGRES_PASSWORD=shumai_password \
|
||||
-e POSTGRES_DB=shumai_db \
|
||||
-p 5432:5432 \
|
||||
-d pgvector/pgvector:pg18
|
||||
```
|
||||
|
||||
#### Step 2: Create a workspace folder
|
||||
|
||||
Create a dedicated directory to store your environment configuration and media files (which are saved in a `./data` directory by default):
|
||||
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
|
||||
#### Step 3: Install Platform-Specific Dependencies
|
||||
|
||||
Make sure the following system dependencies are installed on your host machine before installing Shumai:
|
||||
|
||||
| Package | Description | Ubuntu/Debian | Fedora | Arch | macOS |
|
||||
| :------------- | :------------------------------------------------------ | :------------------------------- | :------------------------------- | :-------------------------------------- | :--------------------- |
|
||||
| **ffmpeg** | Media transcoding and metadata extraction | `sudo apt install -y ffmpeg` | `sudo dnf install -y ffmpeg` | `sudo pacman -S --noconfirm ffmpeg` | `brew install ffmpeg` |
|
||||
| **bubblewrap** | Sandboxing environment for secure AI agent execution | `sudo apt install -y bubblewrap` | `sudo dnf install -y bubblewrap` | `sudo pacman -S --noconfirm bubblewrap` | _NOT REQUIRED_ |
|
||||
| **socat** | Bidirectional socket relay for sandbox network bridging | `sudo apt install -y socat` | `sudo dnf install -y socat` | `sudo pacman -S --noconfirm socat` | `brew install socat` |
|
||||
| **ripgrep** | Fast search tool for workspace security policies | `sudo apt install -y ripgrep` | `sudo dnf install -y ripgrep` | `sudo pacman -S --noconfirm ripgrep` | `brew install ripgrep` |
|
||||
|
||||
> [!NOTE]
|
||||
> **Ubuntu 24.04+ Note:** These releases restrict unprivileged user namespaces by default. To allow `bubblewrap` and the sandbox isolation layer to function, disable this restriction:
|
||||
>
|
||||
> ```bash
|
||||
> sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
> ```
|
||||
>
|
||||
> Alternatively, configure an AppArmor profile that grants user namespace creation (`userns`) privileges to the relevant binaries.
|
||||
|
||||
#### Step 4: Install Shumai globally
|
||||
|
||||
Install Shumai globally using your preferred package manager:
|
||||
|
||||
```bash
|
||||
# NPM
|
||||
npm install -g @shumai-one/shumai
|
||||
|
||||
# PNPM
|
||||
pnpm add -g @shumai-one/shumai
|
||||
|
||||
# Bun
|
||||
bun add -g @shumai-one/shumai
|
||||
```
|
||||
|
||||
#### Step 5: Configure Environment Variables
|
||||
|
||||
Create a `.env` file in your workspace folder (`shumai/`) and add the following configuration:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
BETTER_AUTH_SECRET=ySxs7DxzHDZBbeeHNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
STORAGE_BACKEND=local
|
||||
SHUMAI_SERVER_PORT=3000
|
||||
AWS_ENDPOINT_URL_S3=http://localhost:3000
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> `SHUMAI_SERVER_PORT` sets the port the server starts on, while `AWS_ENDPOINT_URL_S3` is used by the browser to build file upload URLs.
|
||||
> If deploying on a remote server (e.g. `http://123.456.7.8`) with a mapped port (e.g. `12345:3000` in docker-compose), set `AWS_ENDPOINT_URL_S3` to `http://123.456.7.8:12345`.
|
||||
|
||||
#### Step 6: Run Shumai
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **For Bun Users:** Since the published package binaries contain a `#!/usr/bin/env node` shebang, running the global command directly (e.g., `shumai`) will execute under Node.js. If you installed with Bun and want to run under the Bun runtime, you must prefix all commands with `bun run --bun` (e.g., `bun run --bun shumai`, `bun run --bun shumai -d`, `bun run --bun shumai stop`).
|
||||
|
||||
Start the application from your workspace folder:
|
||||
|
||||
```bash
|
||||
shumai
|
||||
```
|
||||
|
||||
Alternatively, you can run and manage Shumai in daemon mode:
|
||||
|
||||
- **Start in daemon mode**:
|
||||
```bash
|
||||
shumai -d
|
||||
```
|
||||
- **Stop Shumai**:
|
||||
```bash
|
||||
shumai stop
|
||||
```
|
||||
- **Restart Shumai**:
|
||||
```bash
|
||||
shumai restart
|
||||
```
|
||||
- **Show/tail logs**:
|
||||
```bash
|
||||
shumai logs
|
||||
```
|
||||
|
||||
On startup, Shumai will automatically run database migrations and start the web server at `http://localhost:3000`.
|
||||
|
||||
---
|
||||
|
||||
### Option 3: Run from Source (Development)
|
||||
|
||||
To set up Shumai locally for development:
|
||||
|
||||
1. Clone the repository and install dependencies:
|
||||
```bash
|
||||
git clone https://github.com/shumaiOne/shumai.git
|
||||
cd shumai
|
||||
bun install
|
||||
```
|
||||
2. Start the `pgvector` database container (as described in **Option 2, Step 1**).
|
||||
3. Create a `.env` file at the root of the workspace using the configuration from **Option 2, Step 5**.
|
||||
4. Apply the database schema migrations:
|
||||
```bash
|
||||
bun run db:migrate
|
||||
```
|
||||
5. Start the local development server:
|
||||
```bash
|
||||
bun run dev
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Command Line Interface (CLI)
|
||||
|
||||
Shumai provides a Command Line Interface (CLI) tool to manage projects, folders, and assets, upload files/folders, and create new versions directly from your terminal.
|
||||
|
||||
For more details on installation and usage, see the [CLI Readme](apps/cli/README.md).
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.0.2",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -4,6 +4,12 @@ loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { TaskQueueAgent } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Agent worker evaluated successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
async function run() {
|
||||
// Initialize workflows and activities
|
||||
@@ -23,7 +29,7 @@ async function run() {
|
||||
await workflowService.startWorkers(TaskQueueAgent, options)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
handleDaemonCommands('shumai-agent', run).catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# Shumai CLI
|
||||
|
||||
A Command Line Interface (CLI) tool for interacting with your Shumai instance.
|
||||
|
||||
The CLI allows you to view projects, list folder contents, create folders, upload files and directories, and create new versions of existing files directly from the terminal.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
The CLI package is bundled inside the Shumai workspace.
|
||||
|
||||
### Environment Setup
|
||||
|
||||
Before using the CLI, you must set the following environment variables to authenticate with your Shumai instance:
|
||||
|
||||
```bash
|
||||
# The endpoint of your Shumai API server (e.g. http://localhost:3000)
|
||||
export SHUMAI_API_SERVER="http://localhost:3000"
|
||||
|
||||
# Your personal API Token (Generate this in the WebUI via Settings -> Developer tab)
|
||||
export SHUMAI_API_KEY="your-developer-api-token"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **For Bun Users:** Since the package binary contains a `#!/usr/bin/env node` shebang, running `shumai-cli` directly will run under Node.js. If you wish to run it using Bun, prefix the command with `bun run --bun` (e.g., `bun run --bun shumai-cli project ls`).
|
||||
|
||||
Run `shumai-cli` or query specific commands:
|
||||
|
||||
```bash
|
||||
shumai-cli <command> [arguments] [options]
|
||||
```
|
||||
|
||||
### Commands
|
||||
|
||||
#### 1. List Projects
|
||||
|
||||
List all projects you have access to. Returns the project ID, name, and its root folder ID.
|
||||
|
||||
```bash
|
||||
shumai-cli project ls
|
||||
```
|
||||
|
||||
#### 2. List Directory Contents
|
||||
|
||||
List direct children (folders and files) of a parent folder.
|
||||
|
||||
```bash
|
||||
shumai-cli ls -p <parentId>
|
||||
```
|
||||
|
||||
- **Options:**
|
||||
- `-p`, `--parent <id>`: The parent folder ID (required).
|
||||
|
||||
#### 3. Create a New Folder
|
||||
|
||||
Create a new folder under a parent folder.
|
||||
|
||||
```bash
|
||||
shumai-cli mkdir <name> -p <parentId>
|
||||
```
|
||||
|
||||
- **Options:**
|
||||
- `-p`, `--parent <id>`: The parent folder ID (required).
|
||||
|
||||
#### 4. Upload Files or Folders
|
||||
|
||||
Upload a local file or folder to a target parent folder. File sizes and paths are mapped, and uploads are streamed chunk-by-chunk with a progress bar.
|
||||
|
||||
```bash
|
||||
shumai-cli upload <localPath> -p <parentId>
|
||||
```
|
||||
|
||||
- **Options:**
|
||||
- `-p`, `--parent <id>`: The parent folder ID (required).
|
||||
|
||||
#### 5. Create a New Version of a File
|
||||
|
||||
Upload a local file as a new version of an existing asset (which must be a file or a version stack). This wraps the assets into a version stack dynamically on the server.
|
||||
|
||||
```bash
|
||||
shumai-cli create-version <localFilePath> -p <parentAssetId>
|
||||
```
|
||||
|
||||
- **Options:**
|
||||
- `-p`, `--parent <id>`: The parent asset ID (file or version stack) to version (required).
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
To run the CLI in development mode inside the repository source code:
|
||||
|
||||
```bash
|
||||
bun run apps/cli/src/index.ts project ls
|
||||
```
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"shumai-cli": "./src/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@shumai/api": "workspace:*",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@shumai/tableprinter": "workspace:*",
|
||||
"hono": "4.12.23",
|
||||
"ulid": "^3.0.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { hc } from 'hono/client'
|
||||
import type { AppType } from '@shumai/api'
|
||||
|
||||
export function getClient() {
|
||||
const apiKey = process.env.SHUMAI_API_KEY
|
||||
const apiServer = process.env.SHUMAI_API_SERVER
|
||||
|
||||
if (!apiServer) {
|
||||
console.error('Error: SHUMAI_API_SERVER environment variable is not set.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (!apiKey) {
|
||||
console.error(
|
||||
'Error: SHUMAI_API_KEY environment variable is not set. Please generate a token in the Settings -> Developer tab.',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
return hc<AppType>(apiServer, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { projectLs } from './project'
|
||||
import { ls } from './ls'
|
||||
import { mkdir } from './mkdir'
|
||||
import { getClient } from '../client'
|
||||
import { upload } from './upload'
|
||||
import { createVersion } from './create-version'
|
||||
import fs from 'node:fs'
|
||||
import http from 'node:http'
|
||||
|
||||
vi.mock('../client', () => ({
|
||||
getClient: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('node:http', () => ({
|
||||
default: {
|
||||
request: vi.fn(),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('CLI Commands', () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let logSpy: any
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let writeSpy: any
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let errorSpy: any
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let exitSpy: any
|
||||
|
||||
const mockClient = {
|
||||
api: {
|
||||
projects: {
|
||||
$get: vi.fn(),
|
||||
':projectId': {
|
||||
$get: vi.fn(),
|
||||
},
|
||||
},
|
||||
folders: {
|
||||
$post: vi.fn(),
|
||||
':folderId': {
|
||||
$get: vi.fn(),
|
||||
children: {
|
||||
$get: vi.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
teams: {
|
||||
':teamId': {
|
||||
upload: {
|
||||
tasks: {
|
||||
$post: vi.fn(),
|
||||
':taskId': {
|
||||
$patch: vi.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks()
|
||||
logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true)
|
||||
errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
exitSpy = vi
|
||||
.spyOn(process, 'exit')
|
||||
.mockImplementation((code?: string | number | null): never => {
|
||||
throw new Error(`process.exit(${code})`)
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(getClient).mockReturnValue(mockClient as any)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
logSpy.mockRestore()
|
||||
writeSpy.mockRestore()
|
||||
errorSpy.mockRestore()
|
||||
exitSpy.mockRestore()
|
||||
})
|
||||
|
||||
describe('project ls', () => {
|
||||
it('prints list of projects correctly', async () => {
|
||||
mockClient.api.projects.$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [
|
||||
{ id: 'p1', name: 'Proj 1', rootFolder: 'root1' },
|
||||
{ id: 'p2', name: 'Proj 2', rootFolder: 'root2' },
|
||||
],
|
||||
}),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
await projectLs()
|
||||
|
||||
expect(mockClient.api.projects.$get).toHaveBeenCalledWith({
|
||||
query: { first: '200' },
|
||||
})
|
||||
expect(writeSpy).toHaveBeenCalledWith(expect.stringContaining('Proj 1'))
|
||||
expect(writeSpy).toHaveBeenCalledWith(expect.stringContaining('Proj 2'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('ls', () => {
|
||||
it('queries folders and files and displays table', async () => {
|
||||
mockClient.api.folders[':folderId'].children.$get
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ id: 'subfolder1', name: 'Sub 1', type: 'folder', sizeByte: 0 }],
|
||||
}),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
data: [{ id: 'file1', name: 'doc.pdf', type: 'file', sizeByte: 1024 }],
|
||||
}),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
await ls('parentId123')
|
||||
|
||||
expect(mockClient.api.folders[':folderId'].children.$get).toHaveBeenCalledTimes(2)
|
||||
expect(writeSpy).toHaveBeenCalledWith(expect.stringContaining('Sub 1'))
|
||||
expect(writeSpy).toHaveBeenCalledWith(expect.stringContaining('doc.pdf'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('mkdir', () => {
|
||||
it('calls POST /folders and prints the folder ID', async () => {
|
||||
mockClient.api.folders.$post.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'new_folder_ulid' }),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
await mkdir('NewFolder', 'parent_ulid')
|
||||
|
||||
expect(mockClient.api.folders.$post).toHaveBeenCalledWith({
|
||||
json: { name: 'NewFolder', parentId: 'parent_ulid' },
|
||||
})
|
||||
expect(logSpy).toHaveBeenCalledWith('Created folder with ID: new_folder_ulid')
|
||||
})
|
||||
})
|
||||
|
||||
describe('upload', () => {
|
||||
it('uploads a file successfully and confirms the task', async () => {
|
||||
const tempFile = 'temp-test-file.txt'
|
||||
fs.writeFileSync(tempFile, 'hello world')
|
||||
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ projectId: 'p123' }),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
mockClient.api.projects[':projectId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ teamId: 't123' }),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
mockClient.api.teams[':teamId'].upload.tasks.$post.mockImplementation(
|
||||
async (req: unknown) => {
|
||||
const rootNodeId = (req as { json: { files: { id: string }[] } }).json.files[0].id
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
taskId: 'task123',
|
||||
presignedUrls: [
|
||||
{ id: rootNodeId, fileId: 'file-123', url: 'http://localhost/upload' },
|
||||
],
|
||||
createdAssets: [{ tempId: rootNodeId, assetId: 'file-123' }],
|
||||
}),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any
|
||||
},
|
||||
)
|
||||
|
||||
mockClient.api.teams[':teamId'].upload.tasks[':taskId'].$patch.mockResolvedValue({
|
||||
ok: true,
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
const mockRes = {
|
||||
statusCode: 200,
|
||||
resume: vi.fn(),
|
||||
}
|
||||
let mockReqCallback: ((res: { statusCode: number; resume: () => void }) => void) | null = null
|
||||
const mockReq = {
|
||||
on: vi.fn().mockImplementation(() => {
|
||||
return mockReq
|
||||
}),
|
||||
write: vi.fn(),
|
||||
end: vi.fn().mockImplementation(() => {
|
||||
if (mockReqCallback) {
|
||||
mockReqCallback(mockRes)
|
||||
}
|
||||
}),
|
||||
destroy: vi.fn(),
|
||||
}
|
||||
|
||||
vi.mocked(http.request).mockImplementation(((
|
||||
_options: unknown,
|
||||
callback: (res: { statusCode: number; resume: () => void }) => void,
|
||||
) => {
|
||||
mockReqCallback = callback
|
||||
return mockReq
|
||||
// We use any here because we are mocking the http request function in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
}) as any)
|
||||
|
||||
await upload(tempFile, 'folder-123')
|
||||
|
||||
expect(mockClient.api.folders[':folderId'].$get).toHaveBeenCalledWith({
|
||||
param: { folderId: 'folder-123' },
|
||||
})
|
||||
expect(mockClient.api.projects[':projectId'].$get).toHaveBeenCalledWith({
|
||||
param: { projectId: 'p123' },
|
||||
})
|
||||
expect(mockClient.api.teams[':teamId'].upload.tasks.$post).toHaveBeenCalled()
|
||||
expect(mockClient.api.teams[':teamId'].upload.tasks[':taskId'].$patch).toHaveBeenCalledWith({
|
||||
param: { teamId: 't123', taskId: 'task123' },
|
||||
json: { fileId: 'file-123' },
|
||||
})
|
||||
|
||||
try {
|
||||
fs.unlinkSync(tempFile)
|
||||
} catch {
|
||||
// Ignore if file doesn't exist
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('create-version', () => {
|
||||
it('fails if local path is a directory', async () => {
|
||||
const tempDir = 'temp-test-dir'
|
||||
fs.mkdirSync(tempDir, { recursive: true })
|
||||
|
||||
await expect(createVersion(tempDir, 'file-123')).rejects.toThrow('process.exit(1)')
|
||||
|
||||
try {
|
||||
fs.rmdirSync(tempDir)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
})
|
||||
|
||||
it('fails if parent asset is not file or version_stack', async () => {
|
||||
const tempFile = 'temp-test-file-ver.txt'
|
||||
fs.writeFileSync(tempFile, 'hello ver')
|
||||
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ type: 'folder', projectId: 'p123' }),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
await expect(createVersion(tempFile, 'folder-123')).rejects.toThrow('process.exit(1)')
|
||||
|
||||
try {
|
||||
fs.unlinkSync(tempFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
})
|
||||
|
||||
it('succeeds if parent asset is a file', async () => {
|
||||
const tempFile = 'temp-test-file-ver.txt'
|
||||
fs.writeFileSync(tempFile, 'hello ver')
|
||||
|
||||
mockClient.api.folders[':folderId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ type: 'file', projectId: 'p123' }),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
mockClient.api.projects[':projectId'].$get.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ teamId: 't123' }),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
mockClient.api.teams[':teamId'].upload.tasks.$post.mockImplementation(
|
||||
async (req: unknown) => {
|
||||
const rootNodeId = (req as { json: { files: { id: string }[] } }).json.files[0].id
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
taskId: 'task123',
|
||||
presignedUrls: [
|
||||
{ id: rootNodeId, fileId: 'file-123', url: 'http://localhost/upload' },
|
||||
],
|
||||
createdAssets: [{ tempId: rootNodeId, assetId: 'file-123' }],
|
||||
}),
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any
|
||||
},
|
||||
)
|
||||
|
||||
mockClient.api.teams[':teamId'].upload.tasks[':taskId'].$patch.mockResolvedValue({
|
||||
ok: true,
|
||||
// We use any here because we are mocking the hono client response shape in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
const mockRes = {
|
||||
statusCode: 200,
|
||||
resume: vi.fn(),
|
||||
}
|
||||
let mockReqCallback: ((res: { statusCode: number; resume: () => void }) => void) | null = null
|
||||
const mockReq = {
|
||||
on: vi.fn().mockImplementation(() => {
|
||||
return mockReq
|
||||
}),
|
||||
write: vi.fn(),
|
||||
end: vi.fn().mockImplementation(() => {
|
||||
if (mockReqCallback) {
|
||||
mockReqCallback(mockRes)
|
||||
}
|
||||
}),
|
||||
destroy: vi.fn(),
|
||||
}
|
||||
|
||||
vi.mocked(http.request).mockImplementation(((
|
||||
_options: unknown,
|
||||
callback: (res: { statusCode: number; resume: () => void }) => void,
|
||||
) => {
|
||||
mockReqCallback = callback
|
||||
return mockReq
|
||||
// We use any here because we are mocking the http request function in a unit test context
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
}) as any)
|
||||
|
||||
await createVersion(tempFile, 'file-123')
|
||||
|
||||
expect(mockClient.api.folders[':folderId'].$get).toHaveBeenCalledWith({
|
||||
param: { folderId: 'file-123' },
|
||||
})
|
||||
expect(mockClient.api.teams[':teamId'].upload.tasks.$post).toHaveBeenCalledWith({
|
||||
param: { teamId: 't123' },
|
||||
json: expect.objectContaining({ parentId: 'file-123' }),
|
||||
})
|
||||
|
||||
try {
|
||||
fs.unlinkSync(tempFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,155 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { ulid } from 'ulid'
|
||||
import { getClient } from '../client'
|
||||
import { getMediaType, ProgressTracker, uploadFileWithProgress } from './upload'
|
||||
|
||||
export async function createVersion(localPath: string, parentId: string) {
|
||||
if (!localPath) {
|
||||
console.error('Error: Local file path is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Parent ID (-p/--parent) is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const resolvedPath = path.resolve(localPath)
|
||||
if (!fs.existsSync(resolvedPath)) {
|
||||
console.error(`Error: Local path "${resolvedPath}" does not exist.`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const stat = fs.statSync(resolvedPath)
|
||||
if (stat.isDirectory()) {
|
||||
console.error(
|
||||
`Error: Local path "${resolvedPath}" is a directory. create-version only supports uploading single files.`,
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
console.log(`Checking parent asset ${parentId}...`)
|
||||
const assetRes = await client.api.folders[':folderId'].$get({
|
||||
param: { folderId: parentId },
|
||||
})
|
||||
|
||||
if (!assetRes.ok) {
|
||||
const err = (await assetRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error fetching parent asset details: ${err.error || 'Failed'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const parentAsset = await assetRes.json()
|
||||
if (parentAsset.type !== 'file' && parentAsset.type !== 'version_stack') {
|
||||
console.error(
|
||||
`Error: Parent asset must be a file or a version_stack (got "${parentAsset.type}").`,
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const projectId = parentAsset.projectId
|
||||
if (!projectId) {
|
||||
console.error('Error: Parent asset does not belong to any project.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const projectRes = await client.api.projects[':projectId'].$get({
|
||||
param: { projectId },
|
||||
})
|
||||
if (!projectRes.ok) {
|
||||
const err = (await projectRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error fetching project details: ${err.error || 'Failed'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
const project = await projectRes.json()
|
||||
const teamId = project.teamId
|
||||
|
||||
const name = path.basename(resolvedPath)
|
||||
const mediaType = getMediaType(name)
|
||||
|
||||
const fileNode = {
|
||||
name,
|
||||
id: ulid(),
|
||||
size: stat.size,
|
||||
children: [],
|
||||
type: 'file',
|
||||
mediaType,
|
||||
}
|
||||
|
||||
console.log(`Creating new version task...`)
|
||||
|
||||
const taskRes = await client.api.teams[':teamId'].upload.tasks.$post({
|
||||
param: { teamId },
|
||||
json: {
|
||||
parentId,
|
||||
files: [fileNode],
|
||||
},
|
||||
})
|
||||
|
||||
if (!taskRes.ok) {
|
||||
const err = (await taskRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error creating version task: ${err.error || 'Failed'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const { taskId, presignedUrls, createdAssets } = await taskRes.json()
|
||||
if (!presignedUrls || presignedUrls.length === 0) {
|
||||
console.error('Error: Server did not return presigned upload URLs.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const urlObj = presignedUrls[0]
|
||||
const progress = new ProgressTracker()
|
||||
progress.addUpload(urlObj.id, name, stat.size)
|
||||
|
||||
try {
|
||||
await uploadFileWithProgress(urlObj.url, resolvedPath, mediaType, (bytes) => {
|
||||
progress.updateProgress(urlObj.id, bytes)
|
||||
})
|
||||
|
||||
const confirmRes = await client.api.teams[':teamId'].upload.tasks[':taskId'].$patch({
|
||||
param: { teamId, taskId },
|
||||
json: { fileId: urlObj.fileId },
|
||||
})
|
||||
|
||||
if (!confirmRes.ok) {
|
||||
throw new Error('Failed to confirm upload')
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`\nFailed to upload ${name}:`, err instanceof Error ? err.message : err)
|
||||
await client.api.teams[':teamId'].upload.tasks[':taskId']
|
||||
.$patch({
|
||||
param: { teamId, taskId },
|
||||
json: {
|
||||
fileId: urlObj.fileId,
|
||||
errorMessage: err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
})
|
||||
.catch(() => {})
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log('\nVersion upload completed!')
|
||||
|
||||
if (createdAssets && createdAssets.length > 0) {
|
||||
console.log('Created version asset ID:')
|
||||
for (const asset of createdAssets) {
|
||||
if (asset.tempId === fileNode.id) {
|
||||
console.log(`- ${asset.assetId}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('\nError connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { newTablePrinter, withTruncate } from '@shumai/tableprinter'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import { getClient } from '../client'
|
||||
|
||||
function formatSize(bytes: number): string {
|
||||
if (bytes === 0) return '-'
|
||||
const k = 1024
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k))
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]
|
||||
}
|
||||
|
||||
export async function ls(parentId: string) {
|
||||
if (!parentId) {
|
||||
console.error('Error: Parent ID (-p/--parent) is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
const [foldersRes, filesRes] = await Promise.all([
|
||||
client.api.folders[':folderId'].children.$get({
|
||||
param: { folderId: parentId },
|
||||
query: { assetType: 'folder', first: '200' },
|
||||
}),
|
||||
client.api.folders[':folderId'].children.$get({
|
||||
param: { folderId: parentId },
|
||||
query: { assetType: 'file', first: '200' },
|
||||
}),
|
||||
])
|
||||
|
||||
if (!foldersRes.ok) {
|
||||
const err = (await foldersRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Failed to list folders'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (!filesRes.ok) {
|
||||
const err = (await filesRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error: ${err.error || 'Failed to list files'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const foldersData = await foldersRes.json()
|
||||
const filesData = await filesRes.json()
|
||||
|
||||
const folders = foldersData.data || []
|
||||
const files = filesData.data || []
|
||||
|
||||
const items = [
|
||||
...folders.map((f: AssetInfo) => ({ ...f, typeDisplay: 'folder' as const })),
|
||||
...files.map((f: AssetInfo) => ({ ...f, typeDisplay: 'file' as const })),
|
||||
]
|
||||
|
||||
if (items.length === 0) {
|
||||
console.log('No assets found in this folder.')
|
||||
return
|
||||
}
|
||||
|
||||
const isTty = process.stdout.isTTY
|
||||
const maxWidth = process.stdout.columns || 80
|
||||
const tp = newTablePrinter(process.stdout, isTty, maxWidth)
|
||||
|
||||
tp.addHeader(['ID', 'Name', 'Type', 'Size'])
|
||||
|
||||
for (const item of items) {
|
||||
tp.addField(item.id || '', withTruncate(null))
|
||||
tp.addField(item.name || '')
|
||||
tp.addField(item.typeDisplay, withTruncate(null))
|
||||
tp.addField(
|
||||
item.typeDisplay === 'folder' ? '-' : formatSize(item.sizeByte || 0),
|
||||
withTruncate(null),
|
||||
)
|
||||
tp.endRow()
|
||||
}
|
||||
|
||||
tp.render()
|
||||
} catch (err) {
|
||||
console.error('Error connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { getClient } from '../client'
|
||||
|
||||
export async function mkdir(name: string, parentId: string) {
|
||||
if (!name) {
|
||||
console.error('Error: Folder name is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Parent ID (-p/--parent) is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
const res = await client.api.folders.$post({
|
||||
json: { name, parentId },
|
||||
})
|
||||
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({ error: 'Unknown error' }))) as { error?: string }
|
||||
console.error(`Error: ${err.error || 'Failed to create folder'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const created = await res.json()
|
||||
console.log(`Created folder with ID: ${created.id}`)
|
||||
} catch (err) {
|
||||
console.error('Error connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { newTablePrinter } from '@shumai/tableprinter'
|
||||
import { getClient } from '../client'
|
||||
|
||||
export async function projectLs() {
|
||||
const client = getClient()
|
||||
try {
|
||||
const res = await client.api.projects.$get({
|
||||
query: { first: '200' },
|
||||
})
|
||||
|
||||
if (!res.ok) {
|
||||
const err = (await res.json().catch(() => ({ error: 'Unknown error' }))) as { error?: string }
|
||||
console.error(`Error: ${err.error || 'Failed to list projects'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const { data: projects } = await res.json()
|
||||
|
||||
if (projects.length === 0) {
|
||||
console.log('No projects found.')
|
||||
return
|
||||
}
|
||||
|
||||
const isTty = process.stdout.isTTY
|
||||
const maxWidth = process.stdout.columns || 80
|
||||
const tp = newTablePrinter(process.stdout, isTty, maxWidth)
|
||||
|
||||
tp.addHeader(['ID', 'Name', 'Root Folder ID'])
|
||||
|
||||
for (const p of projects) {
|
||||
tp.addField(p.id || '')
|
||||
tp.addField(p.name || '')
|
||||
tp.addField(p.rootFolder || '')
|
||||
tp.endRow()
|
||||
}
|
||||
|
||||
tp.render()
|
||||
} catch (err) {
|
||||
console.error('Error connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
import fs from 'node:fs'
|
||||
import fsPromises from 'node:fs/promises'
|
||||
import http from 'node:http'
|
||||
import https from 'node:https'
|
||||
import path from 'node:path'
|
||||
import { ulid } from 'ulid'
|
||||
import { getClient } from '../client'
|
||||
|
||||
interface CliFileNode {
|
||||
name: string
|
||||
id: string
|
||||
size: number
|
||||
children: CliFileNode[]
|
||||
type: 'file' | 'folder'
|
||||
mediaType?: string
|
||||
}
|
||||
|
||||
export function getMediaType(filename: string): string {
|
||||
const ext = path.extname(filename).toLowerCase()
|
||||
const mimeMap: Record<string, string> = {
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.webp': 'image/webp',
|
||||
'.mp4': 'video/mp4',
|
||||
'.mkv': 'video/x-matroska',
|
||||
'.mov': 'video/quicktime',
|
||||
'.avi': 'video/x-msvideo',
|
||||
'.webm': 'video/webm',
|
||||
'.mp3': 'audio/mpeg',
|
||||
'.wav': 'audio/wav',
|
||||
'.ogg': 'audio/ogg',
|
||||
'.txt': 'text/plain',
|
||||
'.json': 'application/json',
|
||||
'.pdf': 'application/pdf',
|
||||
'.zip': 'application/zip',
|
||||
'.tar': 'application/x-tar',
|
||||
'.gz': 'application/gzip',
|
||||
}
|
||||
return mimeMap[ext] || 'application/octet-stream'
|
||||
}
|
||||
|
||||
async function buildFileTree(
|
||||
localPath: string,
|
||||
fileMap: Map<string, { path: string; name: string; size: number; mediaType: string }>,
|
||||
): Promise<CliFileNode> {
|
||||
const stat = await fsPromises.stat(localPath)
|
||||
const name = path.basename(localPath)
|
||||
const id = ulid()
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
const childrenNames = await fsPromises.readdir(localPath)
|
||||
const children: CliFileNode[] = []
|
||||
for (const childName of childrenNames) {
|
||||
if (childName.startsWith('.')) continue
|
||||
const childNode = await buildFileTree(path.join(localPath, childName), fileMap)
|
||||
children.push(childNode)
|
||||
}
|
||||
return {
|
||||
name,
|
||||
id,
|
||||
size: 0,
|
||||
children,
|
||||
type: 'folder',
|
||||
}
|
||||
} else {
|
||||
const mediaType = getMediaType(name)
|
||||
fileMap.set(id, { path: localPath, name, size: stat.size, mediaType })
|
||||
return {
|
||||
name,
|
||||
id,
|
||||
size: stat.size,
|
||||
children: [],
|
||||
type: 'file',
|
||||
mediaType,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ProgressTracker {
|
||||
private uploads = new Map<string, { name: string; size: number; uploadedBytes: number }>()
|
||||
private isTty = process.stdout.isTTY
|
||||
private linesPrinted = 0
|
||||
|
||||
addUpload(id: string, name: string, size: number) {
|
||||
this.uploads.set(id, { name, size, uploadedBytes: 0 })
|
||||
if (this.isTty) {
|
||||
console.log(`${name.padEnd(30)} [${'░'.repeat(20)}] 0%`)
|
||||
this.linesPrinted++
|
||||
console.log('')
|
||||
this.linesPrinted++
|
||||
}
|
||||
}
|
||||
|
||||
updateProgress(id: string, uploadedBytes: number) {
|
||||
const item = this.uploads.get(id)
|
||||
if (!item) return
|
||||
item.uploadedBytes = uploadedBytes
|
||||
|
||||
if (this.isTty) {
|
||||
process.stdout.write('\x1B[' + this.linesPrinted + 'A')
|
||||
this.linesPrinted = 0
|
||||
for (const upload of this.uploads.values()) {
|
||||
const pct =
|
||||
upload.size > 0
|
||||
? Math.min(Math.round((upload.uploadedBytes / upload.size) * 100), 100)
|
||||
: 100
|
||||
const barLength = 20
|
||||
const filledLength = Math.round((pct / 100) * barLength)
|
||||
const bar = '█'.repeat(filledLength) + '░'.repeat(barLength - filledLength)
|
||||
console.log(`${upload.name.padEnd(30)} [${bar}] ${pct}%`)
|
||||
this.linesPrinted++
|
||||
console.log('')
|
||||
this.linesPrinted++
|
||||
}
|
||||
} else {
|
||||
const pct = item.size > 0 ? Math.min(Math.round((uploadedBytes / item.size) * 100), 100) : 100
|
||||
if (pct === 100 || uploadedBytes === 0 || pct % 25 === 0) {
|
||||
console.log(`Uploading ${item.name}: ${pct}%`)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadFileWithProgress(
|
||||
url: string,
|
||||
filePath: string,
|
||||
contentType: string,
|
||||
onProgress: (bytesUploaded: number) => void,
|
||||
) {
|
||||
const stat = fs.statSync(filePath)
|
||||
const totalSize = stat.size
|
||||
|
||||
const parsedUrl = new URL(url)
|
||||
const isHttps = parsedUrl.protocol === 'https:'
|
||||
const requestModule = isHttps ? https : http
|
||||
|
||||
const options = {
|
||||
method: 'PUT',
|
||||
hostname: parsedUrl.hostname,
|
||||
port: parsedUrl.port || (isHttps ? 443 : 80),
|
||||
path: parsedUrl.pathname + parsedUrl.search,
|
||||
headers: {
|
||||
'Content-Type': contentType,
|
||||
'Content-Length': totalSize.toString(),
|
||||
},
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const req = requestModule.request(options, (res) => {
|
||||
res.resume()
|
||||
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new Error(`Failed to upload to S3: ${res.statusCode} ${res.statusMessage}`))
|
||||
}
|
||||
})
|
||||
|
||||
req.on('error', (err) => {
|
||||
reject(err)
|
||||
})
|
||||
|
||||
const fileStream = fs.createReadStream(filePath)
|
||||
let bytesUploaded = 0
|
||||
|
||||
fileStream.on('data', (chunk) => {
|
||||
bytesUploaded += chunk.length
|
||||
onProgress(bytesUploaded)
|
||||
req.write(chunk)
|
||||
})
|
||||
|
||||
fileStream.on('end', () => {
|
||||
req.end()
|
||||
})
|
||||
|
||||
fileStream.on('error', (err) => {
|
||||
req.destroy()
|
||||
reject(err)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export async function upload(localPath: string, parentId: string) {
|
||||
if (!localPath) {
|
||||
console.error('Error: Local file/folder path is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Parent ID (-p/--parent) is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const resolvedPath = path.resolve(localPath)
|
||||
if (!fs.existsSync(resolvedPath)) {
|
||||
console.error(`Error: Local path "${resolvedPath}" does not exist.`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = getClient()
|
||||
|
||||
try {
|
||||
const parentRes = await client.api.folders[':folderId'].$get({
|
||||
param: { folderId: parentId },
|
||||
})
|
||||
if (!parentRes.ok) {
|
||||
const err = (await parentRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error fetching parent folder details: ${err.error || 'Failed'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
const parentFolder = await parentRes.json()
|
||||
const projectId = parentFolder.projectId
|
||||
if (!projectId) {
|
||||
console.error('Error: Parent folder does not belong to any project.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const projectRes = await client.api.projects[':projectId'].$get({
|
||||
param: { projectId },
|
||||
})
|
||||
if (!projectRes.ok) {
|
||||
const err = (await projectRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error fetching project details: ${err.error || 'Failed'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
const project = await projectRes.json()
|
||||
const teamId = project.teamId
|
||||
|
||||
const fileMap = new Map<
|
||||
string,
|
||||
{ path: string; name: string; size: number; mediaType: string }
|
||||
>()
|
||||
const rootNode = await buildFileTree(resolvedPath, fileMap)
|
||||
|
||||
console.log(`Starting upload task for ${fileMap.size} files...`)
|
||||
|
||||
const taskRes = await client.api.teams[':teamId'].upload.tasks.$post({
|
||||
param: { teamId },
|
||||
json: {
|
||||
parentId,
|
||||
files: [rootNode],
|
||||
},
|
||||
})
|
||||
|
||||
if (!taskRes.ok) {
|
||||
const err = (await taskRes.json().catch(() => ({ error: 'Unknown error' }))) as {
|
||||
error?: string
|
||||
}
|
||||
console.error(`Error creating upload task: ${err.error || 'Failed'}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const { taskId, presignedUrls, createdAssets } = await taskRes.json()
|
||||
|
||||
const progress = new ProgressTracker()
|
||||
for (const urlObj of presignedUrls) {
|
||||
const fileMeta = fileMap.get(urlObj.id)
|
||||
if (fileMeta) {
|
||||
progress.addUpload(urlObj.id, fileMeta.name, fileMeta.size)
|
||||
}
|
||||
}
|
||||
|
||||
const concurrencyLimit = 3
|
||||
const queue = [...presignedUrls]
|
||||
const activeUploads: Promise<void>[] = []
|
||||
|
||||
const uploadNext = async (): Promise<void> => {
|
||||
if (queue.length === 0) return
|
||||
const urlObj = queue.shift()!
|
||||
const fileMeta = fileMap.get(urlObj.id)!
|
||||
|
||||
try {
|
||||
await uploadFileWithProgress(urlObj.url, fileMeta.path, fileMeta.mediaType, (bytes) => {
|
||||
progress.updateProgress(urlObj.id, bytes)
|
||||
})
|
||||
|
||||
const confirmRes = await client.api.teams[':teamId'].upload.tasks[':taskId'].$patch({
|
||||
param: { teamId, taskId },
|
||||
json: { fileId: urlObj.fileId },
|
||||
})
|
||||
|
||||
if (!confirmRes.ok) {
|
||||
throw new Error('Failed to confirm upload')
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`\nFailed to upload ${fileMeta.name}:`,
|
||||
err instanceof Error ? err.message : err,
|
||||
)
|
||||
await client.api.teams[':teamId'].upload.tasks[':taskId']
|
||||
.$patch({
|
||||
param: { teamId, taskId },
|
||||
json: {
|
||||
fileId: urlObj.fileId,
|
||||
errorMessage: err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
|
||||
return uploadNext()
|
||||
}
|
||||
|
||||
for (let i = 0; i < Math.min(concurrencyLimit, presignedUrls.length); i++) {
|
||||
activeUploads.push(uploadNext())
|
||||
}
|
||||
|
||||
await Promise.all(activeUploads)
|
||||
|
||||
console.log('\nUpload completed!')
|
||||
|
||||
if (createdAssets && createdAssets.length > 0) {
|
||||
console.log('Uploaded asset IDs:')
|
||||
const topLevelTempIds = new Set<string>()
|
||||
topLevelTempIds.add(rootNode.id)
|
||||
|
||||
for (const asset of createdAssets) {
|
||||
if (topLevelTempIds.has(asset.tempId)) {
|
||||
console.log(`- ${asset.assetId}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('\nError connecting to API server:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { projectLs } from './commands/project'
|
||||
import { ls } from './commands/ls'
|
||||
import { mkdir } from './commands/mkdir'
|
||||
import { upload } from './commands/upload'
|
||||
import { createVersion } from './commands/create-version'
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Usage: shumai-cli <command> [arguments] [options]
|
||||
|
||||
Commands:
|
||||
project ls List all projects (returns id, name, root folder id)
|
||||
ls -p <parentId> List direct children of a parent folder (returns id, name, type, size)
|
||||
mkdir <name> -p <parentId> Create a new folder in a parent folder
|
||||
upload <path> -p <parentId> Upload a local file or folder to a parent folder
|
||||
create-version <path> -p <id> Create a new version of an existing file or version stack
|
||||
|
||||
Options:
|
||||
-p, --parent <id> The parent folder/asset ID (required for ls, mkdir, upload, create-version)
|
||||
-h, --help Show help details
|
||||
`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2)
|
||||
if (args.length === 0 || args.includes('-h') || args.includes('--help')) {
|
||||
printHelp()
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const cmd = args[0]
|
||||
|
||||
let parentId = ''
|
||||
const parentIdx = args.findIndex((a) => a === '-p' || a === '--parent')
|
||||
if (parentIdx !== -1 && parentIdx + 1 < args.length) {
|
||||
parentId = args[parentIdx + 1]
|
||||
}
|
||||
|
||||
const cleanArgs: string[] = []
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '-p' || args[i] === '--parent') {
|
||||
i++
|
||||
} else {
|
||||
cleanArgs.push(args[i])
|
||||
}
|
||||
}
|
||||
|
||||
if (cmd === 'project' && cleanArgs[1] === 'ls') {
|
||||
await projectLs()
|
||||
} else if (cmd === 'ls') {
|
||||
if (!parentId) {
|
||||
console.error('Error: Option -p/--parent <parentId> is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
await ls(parentId)
|
||||
} else if (cmd === 'mkdir') {
|
||||
const name = cleanArgs[1]
|
||||
if (!name) {
|
||||
console.error(
|
||||
'Error: Folder name is required. Usage: shumai-cli mkdir <folderName> -p <parentId>',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Option -p/--parent <parentId> is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
await mkdir(name, parentId)
|
||||
} else if (cmd === 'upload') {
|
||||
const localPath = cleanArgs[1]
|
||||
if (!localPath) {
|
||||
console.error(
|
||||
'Error: File/folder path is required. Usage: shumai-cli upload <localPath> -p <parentId>',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Option -p/--parent <parentId> is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
await upload(localPath, parentId)
|
||||
} else if (cmd === 'create-version') {
|
||||
const localPath = cleanArgs[1]
|
||||
if (!localPath) {
|
||||
console.error(
|
||||
'Error: Local file path is required. Usage: shumai-cli create-version <localPath> -p <parentAssetId>',
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
if (!parentId) {
|
||||
console.error('Error: Option -p/--parent <parentAssetId> is required.')
|
||||
process.exit(1)
|
||||
}
|
||||
await createVersion(localPath, parentId)
|
||||
} else {
|
||||
console.error(`Error: Unknown command "${cmd}". Run "shumai-cli --help" for usage.`)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Unexpected error:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.0.2",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -3,6 +3,12 @@ loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { TaskQueueTranscode, workflowService } from '@shumai/workflow-core'
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Transcode worker evaluated successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
async function run() {
|
||||
// Initialize workflows and activities
|
||||
@@ -22,7 +28,7 @@ async function run() {
|
||||
await workflowService.startWorkers(TaskQueueTranscode, options)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
handleDaemonCommands('shumai-transcode', run).catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.0.2",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,74 +1,136 @@
|
||||
import { loadEnvConfig } from '@shumai/core/src/env-loader'
|
||||
import { join } from 'node:path'
|
||||
loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
|
||||
import index from '@shumai/webui/index.html'
|
||||
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { app } from '@shumai/api'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { app } from '@shumai/api'
|
||||
|
||||
// Initialize workflows and activities for local executor mode
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
workflowService.start()
|
||||
if (process.env.WORKFLOW_EXECUTOR === 'temporal') {
|
||||
const args = process.argv.slice(2)
|
||||
let workersOption = ''
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--workers') {
|
||||
workersOption = args[i + 1] || ''
|
||||
} else if (args[i].startsWith('--workers=')) {
|
||||
workersOption = args[i].split('=')[1]
|
||||
}
|
||||
}
|
||||
|
||||
const queuesToStart: string[] = []
|
||||
if (workersOption === 'agent' || workersOption === 'ai') {
|
||||
queuesToStart.push('agent_queue')
|
||||
} else if (workersOption === 'transcode') {
|
||||
queuesToStart.push('transcode_queue')
|
||||
} else if (workersOption === 'all') {
|
||||
queuesToStart.push('agent_queue', 'transcode_queue')
|
||||
}
|
||||
|
||||
Promise.all(queuesToStart.map((q) => workflowService.startWorkers(q))).catch(console.error)
|
||||
}
|
||||
|
||||
const server = Bun.serve({
|
||||
port: 3000,
|
||||
fetch: app.fetch,
|
||||
maxRequestBodySize: process.env.MAX_REQUEST_BODY_SIZE
|
||||
? parseInt(process.env.MAX_REQUEST_BODY_SIZE)
|
||||
: 1024 * 1024 * 1024 * 10, // Default 10GB
|
||||
routes: {
|
||||
// Serve index.html for root
|
||||
'/': index,
|
||||
|
||||
// Proxy API requests to Hono
|
||||
'/api/*': app.fetch,
|
||||
'/files/*': app.fetch,
|
||||
|
||||
// Catch-all for SPA routing (fallback to index.html)
|
||||
'/*': index,
|
||||
},
|
||||
development: process.env.NODE_ENV !== 'production',
|
||||
})
|
||||
|
||||
console.log(`🚀 Server running at ${server.url}`)
|
||||
|
||||
const shutdown = () => {
|
||||
console.log('\nShutting down gracefully...')
|
||||
assetService.stopCleanupJob()
|
||||
server.stop(true)
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Web app evaluated successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
process.on('SIGINT', shutdown)
|
||||
process.on('SIGTERM', shutdown)
|
||||
async function run() {
|
||||
// Initialize workflows and activities for local executor mode
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
workflowService.start()
|
||||
if (process.env.WORKFLOW_EXECUTOR === 'temporal') {
|
||||
const args = process.argv.slice(2)
|
||||
let workersOption = ''
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--workers') {
|
||||
workersOption = args[i + 1] || ''
|
||||
} else if (args[i].startsWith('--workers=')) {
|
||||
workersOption = args[i].split('=')[1]
|
||||
}
|
||||
}
|
||||
|
||||
const queuesToStart: string[] = []
|
||||
if (workersOption === 'agent' || workersOption === 'ai') {
|
||||
queuesToStart.push('agent_queue')
|
||||
} else if (workersOption === 'transcode') {
|
||||
queuesToStart.push('transcode_queue')
|
||||
} else if (workersOption === 'all') {
|
||||
queuesToStart.push('agent_queue', 'transcode_queue')
|
||||
}
|
||||
|
||||
Promise.all(queuesToStart.map((q) => workflowService.startWorkers(q))).catch(console.error)
|
||||
}
|
||||
|
||||
const isProd = process.env.NODE_ENV === 'production'
|
||||
|
||||
const port = process.env.SHUMAI_SERVER_PORT ? parseInt(process.env.SHUMAI_SERVER_PORT) : 3000
|
||||
|
||||
interface HtmlBundleFile {
|
||||
input: string
|
||||
path: string
|
||||
loader: string
|
||||
isEntry: boolean
|
||||
headers: Record<string, string>
|
||||
}
|
||||
|
||||
interface HtmlBundle {
|
||||
index: string
|
||||
files: HtmlBundleFile[]
|
||||
}
|
||||
|
||||
// Bun's native Serve config routes require specific handler types that are hard
|
||||
// to dynamically construct without using 'any'.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const routes: Record<string, any> = {
|
||||
'/api/*': app.fetch,
|
||||
'/files/*': app.fetch,
|
||||
}
|
||||
|
||||
if (isProd) {
|
||||
const bundle = index as unknown as HtmlBundle
|
||||
if (bundle && typeof bundle === 'object' && 'files' in bundle && Array.isArray(bundle.files)) {
|
||||
for (const file of bundle.files) {
|
||||
const routePath = file.path.startsWith('.') ? file.path.slice(1) : file.path
|
||||
const filePath = join(import.meta.dir, file.path)
|
||||
routes[routePath] = () => new Response(Bun.file(filePath), { headers: file.headers })
|
||||
}
|
||||
|
||||
const mainHtmlPath = join(import.meta.dir, bundle.index)
|
||||
const mainHtmlFile = Bun.file(mainHtmlPath)
|
||||
const mainHtmlHeaders = bundle.files.find((f) => f.path === bundle.index)?.headers || {
|
||||
'content-type': 'text/html;charset=utf-8',
|
||||
}
|
||||
const serveMainHtml = () => new Response(mainHtmlFile, { headers: mainHtmlHeaders })
|
||||
|
||||
routes['/'] = serveMainHtml
|
||||
routes['/*'] = serveMainHtml
|
||||
} else {
|
||||
const fallbackHtmlPath = join(import.meta.dir, 'shumai-app.html')
|
||||
const serveFallbackHtml = () =>
|
||||
new Response(Bun.file(fallbackHtmlPath), {
|
||||
headers: { 'content-type': 'text/html;charset=utf-8' },
|
||||
})
|
||||
routes['/'] = serveFallbackHtml
|
||||
routes['/*'] = serveFallbackHtml
|
||||
}
|
||||
} else {
|
||||
routes['/'] = index
|
||||
routes['/*'] = index
|
||||
}
|
||||
|
||||
const server = Bun.serve({
|
||||
port,
|
||||
maxRequestBodySize: process.env.MAX_REQUEST_BODY_SIZE
|
||||
? parseInt(process.env.MAX_REQUEST_BODY_SIZE)
|
||||
: 1024 * 1024 * 1024 * 10, // Default 10GB
|
||||
development: !isProd,
|
||||
fetch: app.fetch,
|
||||
routes,
|
||||
})
|
||||
|
||||
console.log(`🚀 Server running at ${server.url}`)
|
||||
|
||||
const shutdown = () => {
|
||||
console.log('\nShutting down gracefully...')
|
||||
assetService.stopCleanupJob()
|
||||
server.stop(true)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
process.on('SIGINT', shutdown)
|
||||
process.on('SIGTERM', shutdown)
|
||||
}
|
||||
|
||||
handleDaemonCommands('shumai', run).catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -20,6 +20,7 @@ await Bun.build({
|
||||
target: 'bun',
|
||||
outdir: outdir,
|
||||
minify: false,
|
||||
naming: '[name].[ext]',
|
||||
plugins: [
|
||||
temporalWorkflow({
|
||||
bundleOptions: {},
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "10.0.1",
|
||||
"@inlang/paraglide-js": "2.20.0",
|
||||
"@testcontainers/postgresql": "12.0.1",
|
||||
"@types/bun": "1.3.14",
|
||||
"@types/node": "25.9.1",
|
||||
@@ -32,7 +33,7 @@
|
||||
},
|
||||
"apps/agent": {
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"bin": {
|
||||
"shumai-agent": "./src/index.ts",
|
||||
},
|
||||
@@ -42,9 +43,24 @@
|
||||
"@shumai/workflow-core": "workspace:*",
|
||||
},
|
||||
},
|
||||
"apps/cli": {
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.1.2",
|
||||
"bin": {
|
||||
"shumai-cli": "./src/index.ts",
|
||||
},
|
||||
"dependencies": {
|
||||
"@shumai/api": "workspace:*",
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
"@shumai/tableprinter": "workspace:*",
|
||||
"hono": "4.12.23",
|
||||
"ulid": "^3.0.2",
|
||||
},
|
||||
},
|
||||
"apps/transcode": {
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"bin": {
|
||||
"shumai-transcode": "./src/index.ts",
|
||||
},
|
||||
@@ -56,7 +72,7 @@
|
||||
},
|
||||
"apps/web": {
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@shumai/agent": "workspace:*",
|
||||
"@shumai/api": "workspace:*",
|
||||
@@ -71,7 +87,7 @@
|
||||
},
|
||||
"packages/agent": {
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "0.0.52",
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
@@ -95,7 +111,7 @@
|
||||
},
|
||||
"packages/api": {
|
||||
"name": "@shumai/api",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@hono/zod-validator": "0.7.6",
|
||||
"@shumai/core": "workspace:*",
|
||||
@@ -109,7 +125,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@shumai/core",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@shumai/db": "workspace:*",
|
||||
"@shumai/dtos": "workspace:*",
|
||||
@@ -123,6 +139,7 @@
|
||||
"hono": "4.12.23",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"pino": "10.3.1",
|
||||
"sharp": "0.34.5",
|
||||
"tweetnacl": "1.0.3",
|
||||
"ulid": "3.0.2",
|
||||
"yaml": "2.9.0",
|
||||
@@ -135,7 +152,7 @@
|
||||
},
|
||||
"packages/db": {
|
||||
"name": "@shumai/db",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "0.78.0",
|
||||
"@prisma/adapter-pg": "7.8.0",
|
||||
@@ -151,14 +168,18 @@
|
||||
},
|
||||
"packages/dtos": {
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"zod": "4.4.3",
|
||||
},
|
||||
},
|
||||
"packages/tableprinter": {
|
||||
"name": "@shumai/tableprinter",
|
||||
"version": "0.1.2",
|
||||
},
|
||||
"packages/transcode": {
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -167,13 +188,12 @@
|
||||
"@temporalio/activity": "1.17.2",
|
||||
"@temporalio/worker": "1.17.2",
|
||||
"@temporalio/workflow": "1.17.2",
|
||||
"sharp": "0.34.5",
|
||||
"ulid": "3.0.2",
|
||||
},
|
||||
},
|
||||
"packages/webui": {
|
||||
"name": "@shumai/webui",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@base-ui/react": "1.5.0",
|
||||
"@dnd-kit/abstract": "0.4.0",
|
||||
@@ -267,13 +287,14 @@
|
||||
"postcss": "8.5.15",
|
||||
"tailwindcss": "4.3.0",
|
||||
"tw-animate-css": "1.3.3",
|
||||
"typescript": "6.0.3",
|
||||
"vite": "7.3.5",
|
||||
"vite-tsconfig-paths": "6.1.1",
|
||||
},
|
||||
},
|
||||
"packages/workflow-core": {
|
||||
"name": "@shumai/workflow-core",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.2",
|
||||
"dependencies": {
|
||||
"@shumai/core": "workspace:*",
|
||||
"@shumai/db": "workspace:*",
|
||||
@@ -586,6 +607,12 @@
|
||||
|
||||
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
|
||||
|
||||
"@inlang/paraglide-js": ["@inlang/paraglide-js@2.20.0", "", { "dependencies": { "@inlang/recommend-sherlock": "^0.2.1", "@inlang/sdk": "^2.10.0", "commander": "11.1.0", "consola": "3.4.0", "json5": "2.2.3", "unplugin": "^2.1.2", "urlpattern-polyfill": "^10.0.0" }, "peerDependencies": { "typescript": ">=5.6" }, "optionalPeers": ["typescript"], "bin": { "paraglide-js": "bin/run.js" } }, "sha512-vI8PdPVZSnpYnpagjvm+nWSa3nMDRJKVM/2eLAtUHFrNcZZadxVdmP79r4W95+dkMLDFGEyzSSY9sFxvjdkkyQ=="],
|
||||
|
||||
"@inlang/recommend-sherlock": ["@inlang/recommend-sherlock@0.2.1", "", { "dependencies": { "comment-json": "^4.2.3" } }, "sha512-ckv8HvHy/iTqaVAEKrr+gnl+p3XFNwe5D2+6w6wJk2ORV2XkcRkKOJ/XsTUJbPSiyi4PI+p+T3bqbmNx/rDUlg=="],
|
||||
|
||||
"@inlang/sdk": ["@inlang/sdk@2.10.2", "", { "dependencies": { "@lix-js/sdk": "0.4.10", "@sinclair/typebox": "^0.31.17", "kysely": "^0.28.12", "sqlite-wasm-kysely": "0.3.0", "uuid": "^14.0.0" } }, "sha512-O1ki72SNK6LPagaGrvlioBb1mWKvump7cO7P85hfGZjdFTmDdn3icI0A6MvaBsB3P9KQHAjzyubnN1OslGufTw=="],
|
||||
|
||||
"@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="],
|
||||
|
||||
"@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="],
|
||||
@@ -682,6 +709,10 @@
|
||||
|
||||
"@lexical/yjs": ["@lexical/yjs@0.42.0", "", { "dependencies": { "@lexical/offset": "0.42.0", "@lexical/selection": "0.42.0", "lexical": "0.42.0" }, "peerDependencies": { "yjs": ">=13.5.22" } }, "sha512-DplzWnYhfFceGPR+UyDFpZdB287wF/vNOHFuDsBF/nGDdTezvr0Gf60opzyBEF3oXym6p3xTmGygxvO97LZ+vw=="],
|
||||
|
||||
"@lix-js/sdk": ["@lix-js/sdk@0.4.10", "", { "dependencies": { "@lix-js/server-protocol-schema": "0.1.1", "dedent": "1.5.1", "human-id": "^4.1.1", "js-sha256": "^0.11.0", "kysely": "^0.28.12", "sqlite-wasm-kysely": "0.3.0", "uuid": "^14.0.0" } }, "sha512-0dMInAJK/67guTG5rRZaCEhvzC5cCXENOjaePA5AqMXrCE97kaY7SRor9e2vnoGsFIiGqXKlT0MCIoZj36G0gg=="],
|
||||
|
||||
"@lix-js/server-protocol-schema": ["@lix-js/server-protocol-schema@0.1.1", "", {}, "sha512-jBeALB6prAbtr5q4vTuxnRZZv1M2rKe8iNqRQhFJ4Tv7150unEa0vKyz0hs8Gl3fUGsWaNJBh3J8++fpbrpRBQ=="],
|
||||
|
||||
"@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="],
|
||||
|
||||
"@noble/ciphers": ["@noble/ciphers@2.2.0", "", {}, "sha512-Z6pjIZ/8IJcCGzb2S/0Px5J81yij85xASuk1teLNeg75bfT07MV3a/O2Mtn1I2se43k3lkVEcFaR10N4cgQcZA=="],
|
||||
@@ -968,12 +999,16 @@
|
||||
|
||||
"@shumai/api": ["@shumai/api@workspace:packages/api"],
|
||||
|
||||
"@shumai/cli": ["@shumai/cli@workspace:apps/cli"],
|
||||
|
||||
"@shumai/core": ["@shumai/core@workspace:packages/core"],
|
||||
|
||||
"@shumai/db": ["@shumai/db@workspace:packages/db"],
|
||||
|
||||
"@shumai/dtos": ["@shumai/dtos@workspace:packages/dtos"],
|
||||
|
||||
"@shumai/tableprinter": ["@shumai/tableprinter@workspace:packages/tableprinter"],
|
||||
|
||||
"@shumai/transcode": ["@shumai/transcode@workspace:packages/transcode"],
|
||||
|
||||
"@shumai/transcode-app": ["@shumai/transcode-app@workspace:apps/transcode"],
|
||||
@@ -1004,6 +1039,8 @@
|
||||
|
||||
"@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@sqlite.org/sqlite-wasm": ["@sqlite.org/sqlite-wasm@3.48.0-build4", "", { "bin": { "sqlite-wasm": "bin/index.js" } }, "sha512-hI6twvUkzOmyGZhQMza1gpfqErZxXRw6JEsiVjUbo7tFanVD+8Oil0Ih3l2nGzHdxPI41zFmfUQG7GHqhciKZQ=="],
|
||||
|
||||
"@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
|
||||
|
||||
"@standard-schema/utils": ["@standard-schema/utils@0.3.0", "", {}, "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g=="],
|
||||
@@ -1322,6 +1359,8 @@
|
||||
|
||||
"array-includes": ["array-includes@3.1.9", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "define-properties": "^1.2.1", "es-abstract": "^1.24.0", "es-object-atoms": "^1.1.1", "get-intrinsic": "^1.3.0", "is-string": "^1.1.1", "math-intrinsics": "^1.1.0" } }, "sha512-FmeCCAenzH0KH381SPT5FZmiA/TmpndpcaShhfgEN9eCVjnFBqq3l1xrI42y8+PPLI6hypzou4GXw00WHmPBLQ=="],
|
||||
|
||||
"array-timsort": ["array-timsort@1.0.3", "", {}, "sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ=="],
|
||||
|
||||
"array.prototype.findlast": ["array.prototype.findlast@1.2.5", "", { "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", "es-abstract": "^1.23.2", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0", "es-shim-unscopables": "^1.0.2" } }, "sha512-CVvd6FHg1Z3POpBLxO6E6zr+rSKEQ9L6rZHAaY7lLfhKsWYUBBOuMs0e9o24oopj6H+geRCX0YJ+TJLBK2eHyQ=="],
|
||||
|
||||
"array.prototype.flat": ["array.prototype.flat@1.3.3", "", { "dependencies": { "call-bind": "^1.0.8", "define-properties": "^1.2.1", "es-abstract": "^1.23.5", "es-shim-unscopables": "^1.0.2" } }, "sha512-rwG/ja1neyLqCuGZ5YYrznA62D4mZXg0i1cIskIUKSiqF3Cje9/wXAls9B9s1Wa2fomMsIv8czB8jZcPmxCXFg=="],
|
||||
@@ -1458,7 +1497,9 @@
|
||||
|
||||
"comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="],
|
||||
|
||||
"commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="],
|
||||
"commander": ["commander@11.1.0", "", {}, "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ=="],
|
||||
|
||||
"comment-json": ["comment-json@4.6.2", "", { "dependencies": { "array-timsort": "^1.0.3", "esprima": "^4.0.1" } }, "sha512-R2rze/hDX30uul4NZoIZ76ImSJLFxn/1/ZxtKC1L77y2X1k+yYu1joKbAtMA2Fg3hZrTOiw0I5mwVMo0cf250w=="],
|
||||
|
||||
"compress-commons": ["compress-commons@6.0.2", "", { "dependencies": { "crc-32": "^1.2.0", "crc32-stream": "^6.0.0", "is-stream": "^2.0.1", "normalize-path": "^3.0.0", "readable-stream": "^4.0.0" } }, "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg=="],
|
||||
|
||||
@@ -1466,6 +1507,8 @@
|
||||
|
||||
"confbox": ["confbox@0.2.4", "", {}, "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ=="],
|
||||
|
||||
"consola": ["consola@3.4.0", "", {}, "sha512-EiPU8G6dQG0GFHNR8ljnZFki/8a+cQwEQ+7wpxdChl02Q8HXlwEZWD5lqAF8vC2sEC3Tehr8hy7vErz88LHyUA=="],
|
||||
|
||||
"convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="],
|
||||
|
||||
"cookie-es": ["cookie-es@3.1.1", "", {}, "sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg=="],
|
||||
@@ -1522,6 +1565,8 @@
|
||||
|
||||
"decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="],
|
||||
|
||||
"dedent": ["dedent@1.5.1", "", { "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "optionalPeers": ["babel-plugin-macros"] }, "sha512-+LxW+KLWxu3HW3M2w2ympwtqPrqYRzU8fqi6Fhd18fBALe15blJPI/I4+UHveMVG6lJqB4JNd4UG0S5cnVHwIg=="],
|
||||
|
||||
"deep-is": ["deep-is@0.1.4", "", {}, "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="],
|
||||
|
||||
"deepmerge-ts": ["deepmerge-ts@7.1.5", "", {}, "sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw=="],
|
||||
@@ -1626,6 +1671,8 @@
|
||||
|
||||
"espree": ["espree@11.2.0", "", { "dependencies": { "acorn": "^8.16.0", "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^5.0.1" } }, "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw=="],
|
||||
|
||||
"esprima": ["esprima@4.0.1", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="],
|
||||
|
||||
"esquery": ["esquery@1.7.0", "", { "dependencies": { "estraverse": "^5.1.0" } }, "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g=="],
|
||||
|
||||
"esrecurse": ["esrecurse@4.3.0", "", { "dependencies": { "estraverse": "^5.2.0" } }, "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag=="],
|
||||
@@ -1800,6 +1847,8 @@
|
||||
|
||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
||||
|
||||
"human-id": ["human-id@4.2.0", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-K3GbkIWqyvvlpfhBPlbEvD97TtqBpAYA4kt+cn2lD2x2HuohzZCibcA2nOlnJT6exqvJLggoB5nv2dNf192nEA=="],
|
||||
|
||||
"hyperdyperid": ["hyperdyperid@1.2.0", "", {}, "sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
|
||||
@@ -1912,6 +1961,8 @@
|
||||
|
||||
"joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="],
|
||||
|
||||
"js-sha256": ["js-sha256@0.11.1", "", {}, "sha512-o6WSo/LUvY2uC4j7mO50a2ms7E/EAdbP0swigLV+nzHKTTaYnaLIWJ02VdXrsJX0vGedDESQnLsOekr94ryfjg=="],
|
||||
|
||||
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
|
||||
"jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
|
||||
@@ -2400,6 +2451,8 @@
|
||||
|
||||
"split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
|
||||
|
||||
"sqlite-wasm-kysely": ["sqlite-wasm-kysely@0.3.0", "", { "dependencies": { "@sqlite.org/sqlite-wasm": "^3.48.0-build2" }, "peerDependencies": { "kysely": "*" } }, "sha512-TzjBNv7KwRw6E3pdKdlRyZiTmUIE0UttT/Sl56MVwVARl/u5gp978KepazCJZewFUnlWHz9i3NQd4kOtP/Afdg=="],
|
||||
|
||||
"sqlstring": ["sqlstring@2.3.3", "", {}, "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg=="],
|
||||
|
||||
"ssh-remote-port-forward": ["ssh-remote-port-forward@1.0.4", "", { "dependencies": { "@types/ssh2": "^0.5.48", "ssh2": "^1.4.0" } }, "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ=="],
|
||||
@@ -2550,12 +2603,14 @@
|
||||
|
||||
"unist-util-visit-parents": ["unist-util-visit-parents@6.0.2", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ=="],
|
||||
|
||||
"unplugin": ["unplugin@3.0.0", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "picomatch": "^4.0.3", "webpack-virtual-modules": "^0.6.2" } }, "sha512-0Mqk3AT2TZCXWKdcoaufeXNukv2mTrEZExeXlHIOZXdqYoHHr4n51pymnwV8x2BOVxwXbK2HLlI7usrqMpycdg=="],
|
||||
"unplugin": ["unplugin@2.3.11", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "acorn": "^8.15.0", "picomatch": "^4.0.3", "webpack-virtual-modules": "^0.6.2" } }, "sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww=="],
|
||||
|
||||
"update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="],
|
||||
|
||||
"uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="],
|
||||
|
||||
"urlpattern-polyfill": ["urlpattern-polyfill@10.1.0", "", {}, "sha512-IGjKp/o0NL3Bso1PymYURCJxMPNAf/ILOpendP9f5B6e1rTJgdgiOvgfoT8VxCAdY+Wisb9uhGaJJf3yZ2V9nw=="],
|
||||
|
||||
"use-callback-ref": ["use-callback-ref@1.3.3", "", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg=="],
|
||||
|
||||
"use-image": ["use-image@1.1.4", "", { "peerDependencies": { "react": ">=16.8.0", "react-dom": ">=16.8.0" } }, "sha512-P+swhszzHHgEb2X2yQ+vQNPCq/8Ks3hyfdXAVN133pvnvK7UK++bUaZUa5E+A3S02Mw8xOCBr9O6CLhk2fjrWA=="],
|
||||
@@ -2656,6 +2711,8 @@
|
||||
|
||||
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
|
||||
|
||||
"@anthropic-ai/sandbox-runtime/commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="],
|
||||
|
||||
"@anthropic-ai/sandbox-runtime/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
|
||||
|
||||
"@aws-sdk/client-bedrock-runtime/@smithy/node-http-handler": ["@smithy/node-http-handler@4.7.6", "", { "dependencies": { "@smithy/core": "^3.24.6", "@smithy/types": "^4.14.3", "tslib": "^2.6.2" } }, "sha512-3fya8i7GrJilQouk4cZJKdy5k8MWQBpjfXrRNaXDedH8r779tr0jcxyH3+yoTmsluc2+vF4S343yFbnvu8ExDQ=="],
|
||||
@@ -2674,6 +2731,10 @@
|
||||
|
||||
"@grpc/grpc-js/@grpc/proto-loader": ["@grpc/proto-loader@0.8.1", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg=="],
|
||||
|
||||
"@inlang/sdk/@sinclair/typebox": ["@sinclair/typebox@0.31.28", "", {}, "sha512-/s55Jujywdw/Jpan+vsy6JZs1z2ZTGxTmbZTPiuSL2wz9mfzA2gN1zzaqmvfi4pq+uOt7Du85fkiwv5ymW84aQ=="],
|
||||
|
||||
"@inlang/sdk/uuid": ["uuid@14.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg=="],
|
||||
|
||||
"@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
|
||||
|
||||
"@isaacs/cliui/strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="],
|
||||
@@ -2688,6 +2749,8 @@
|
||||
|
||||
"@jsonjoy.com/util/@jsonjoy.com/buffers": ["@jsonjoy.com/buffers@1.2.1", "", { "peerDependencies": { "tslib": "2" } }, "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA=="],
|
||||
|
||||
"@lix-js/sdk/uuid": ["uuid@14.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg=="],
|
||||
|
||||
"@prisma/dev/std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="],
|
||||
|
||||
"@prisma/engines/@prisma/get-platform": ["@prisma/get-platform@7.8.0", "", { "dependencies": { "@prisma/debug": "7.8.0" } }, "sha512-WlxgRGnolL8VH2EmkH1R/DkKNr/mVdS3G2h42IZFFZ3eUrH9OT6t73kIOSlkkrv50wG123Iq8d96ufv5LlZktw=="],
|
||||
@@ -2892,6 +2955,8 @@
|
||||
|
||||
"@tanstack/react-router/@tanstack/react-store": ["@tanstack/react-store@0.9.3", "", { "dependencies": { "@tanstack/store": "0.9.3", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-y2iHd/N9OkoQbFJLUX1T9vbc2O9tjH0pQRgTcx1/Nz4IlwLvkgpuglXUx+mXt0g5ZDFrEeDnONPqkbfxXJKwRg=="],
|
||||
|
||||
"@tanstack/router-plugin/unplugin": ["unplugin@3.0.0", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "picomatch": "^4.0.3", "webpack-virtual-modules": "^0.6.2" } }, "sha512-0Mqk3AT2TZCXWKdcoaufeXNukv2mTrEZExeXlHIOZXdqYoHHr4n51pymnwV8x2BOVxwXbK2HLlI7usrqMpycdg=="],
|
||||
|
||||
"@temporalio/common/ms": ["ms@3.0.0-canary.1", "", {}, "sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g=="],
|
||||
|
||||
"@temporalio/proto/protobufjs": ["protobufjs@7.5.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg=="],
|
||||
|
||||
@@ -17,22 +17,25 @@ services:
|
||||
retries: 5
|
||||
|
||||
shumai:
|
||||
image: shumaione/shumai:main
|
||||
image: shumaione/shumai:latest
|
||||
container_name: shumai_app
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ../../data:/app/data
|
||||
- ./data:/app/data
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_URL: postgres://shumai:shumai_password@shumai_postgres:5432/shumai_db?search_path=public&sslmode=disable
|
||||
|
||||
# Server Configuration
|
||||
SHUMAI_SERVER_PORT: 3000
|
||||
|
||||
# S3 Configuration
|
||||
STORAGE_BACKEND: local
|
||||
AWS_ENDPOINT_URL_S3: http://localhost:3000
|
||||
|
||||
# Better Auth
|
||||
BETTER_AUTH_SECRET: qSxs9DxzHDZBbeeTNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
BETTER_AUTH_URL: https://staging.shumai.one/
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
|
||||
@@ -33,7 +33,6 @@ services:
|
||||
S3_ACCESS_KEY_ID: ID
|
||||
S3_SECRET_ACCESS_KEY: KEY
|
||||
BETTER_AUTH_SECRET: qSxs9DxzHDZBbeeTNPEwBuspYwipBqz5Gk5XdBjNhWw=
|
||||
BETTER_AUTH_URL: http://localhost:3000
|
||||
WORKFLOW_EXECUTOR: temporal
|
||||
TEMPORAL_ADDRESS: temporal:7233
|
||||
GEMINI_API_KEY: KEY
|
||||
|
||||
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 93 KiB |
@@ -0,0 +1,95 @@
|
||||
---
|
||||
title: Environment Variables
|
||||
description: Comprehensive configuration reference for Shumai.
|
||||
---
|
||||
|
||||
Shumai is configured via environment variables. You can define these variables in a `.env` file at the root of your application workspace, or pass them directly to your Docker containers.
|
||||
|
||||
---
|
||||
|
||||
## Database Config
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------- | :----------------------------------------------------------------------------------------- | :------------------------------------------------------- |
|
||||
| `DATABASE_URL` | PostgreSQL connection URL. Must target a database with the `pgvector` extension installed. | `postgresql://user:pass@localhost:5432/db?schema=public` |
|
||||
|
||||
---
|
||||
|
||||
## Server Config
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :---------------------- | :------------------------------------------------------------------------------------------------ | :---------------- |
|
||||
| `SHUMAI_SERVER_PORT` | The port on which the Shumai web server will listen. | `3000` (default) |
|
||||
| `MAX_REQUEST_BODY_SIZE` | Maximum allowed size for request bodies (e.g., for large file uploads). Default is 10GB in bytes. | `10737418240` |
|
||||
|
||||
---
|
||||
|
||||
## Authentication Config
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------- | :--------------------------------------------------------------------- | :--------------------------------------------------------------- |
|
||||
| `BETTER_AUTH_SECRET` | A secure, random 32-byte secret used to sign session cookies and JWTs. | Generate with `openssl rand -base64 32` |
|
||||
| `BETTER_AUTH_URL` | The public URL of your Hono backend API. Used for redirect flows. | `http://localhost:3000` (respects `SHUMAI_SERVER_PORT` fallback) |
|
||||
|
||||
---
|
||||
|
||||
## Storage Config
|
||||
|
||||
Shumai supports storing files locally or on S3-compatible endpoints.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | :------------------------------------------------------------------------ |
|
||||
| `STORAGE_BACKEND` | Storage type. Use `local` for local file system storage, or `s3` for S3-compatible cloud storage. | `local` (default) or `s3` |
|
||||
| `AWS_ENDPOINT_URL_S3` | Endpoint for S3 storage or local storage. Used by the browser to build file upload URLs. Must include port if deploying remotely or using custom ports. | `http://localhost:3000` (respects `SHUMAI_SERVER_PORT` fallback if unset) |
|
||||
| `S3_BUCKET` | S3 bucket name. | `shumai` (default) |
|
||||
| `S3_ACCESS_KEY_ID` | Access key credentials for S3. | `minioadmin` (default) |
|
||||
| `S3_SECRET_ACCESS_KEY` | Secret key credentials for S3. | `minioadmin` (default) |
|
||||
| `S3_REGION` | S3 storage region. | `auto` (default) |
|
||||
| `S3_USE_SSL` | Set to `true` to use HTTPS for S3 endpoints. | `false` (default) |
|
||||
| `PRESIGNED_URL_EXPIRES_IN` | Number of hours for which generated presigned download/upload links are valid. | `5` (default) |
|
||||
|
||||
### Local Storage Port Configuration
|
||||
|
||||
When `STORAGE_BACKEND` is set to `local` (default):
|
||||
|
||||
- `SHUMAI_SERVER_PORT` configures the port on which the Shumai web server starts and listens.
|
||||
- `AWS_ENDPOINT_URL_S3` configures the public/external endpoint used by client browsers to upload and download files. It must include the scheme and port.
|
||||
|
||||
For example, if you deploy Shumai on a remote server at `http://123.456.7.8` and configure port mapping `12345:3000` (exposing port 3000 internally as port 12345 externally) in Docker Compose:
|
||||
|
||||
- Set `SHUMAI_SERVER_PORT` to `3000`.
|
||||
- Set `AWS_ENDPOINT_URL_S3` to `http://123.456.7.8:12345`.
|
||||
|
||||
---
|
||||
|
||||
## Workflow Config
|
||||
|
||||
Configures whether background transcoding and AI agents are executed locally or offloaded to a Temporal cluster.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------------- | :------------------------------------------------------------------------------------------------- | :------------------------------ |
|
||||
| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` |
|
||||
| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) |
|
||||
| `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) |
|
||||
| `EMBEDDING_CHUNK_OVERLAP` | The duration in seconds to overlap video chunks when generating video embeddings. | `5.0` (default) |
|
||||
|
||||
---
|
||||
|
||||
## Logging Config
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :----------- | :--------------------------------------------------------------------------------------------------- | :---------------- |
|
||||
| `LOG_LEVEL` | Pino logging level. Options: `debug`, `info`, `warn`, `error`. | `info` (default) |
|
||||
| `LOG_FORMAT` | Log format style. Use `json` for production logging, or `pretty` for human-readable terminal output. | `json` (default) |
|
||||
|
||||
---
|
||||
|
||||
## LLM API Key
|
||||
|
||||
API keys to AI providers on startup.
|
||||
|
||||
| Variable | Description | Example / Default |
|
||||
| :------------------ | :---------------------------------- | :---------------- |
|
||||
| `GEMINI_API_KEY` | API key for Google Gemini provider. | `AIzaSy...` |
|
||||
| `OPENAI_API_KEY` | API key for OpenAI provider. | `sk-proj-...` |
|
||||
| `ANTHROPIC_API_KEY` | API key for Anthropic provider. | `sk-ant-...` |
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
title: Deploy with Temporal
|
||||
description: Configure a high-scale production deployment using Temporal and S3-compatible storage.
|
||||
---
|
||||
|
||||
For production environments or workloads with high-volume video processing and heavy AI agent usage, running background tasks on a single server can cause bottlenecks. Shumai resolves this by offloading task queue management, retries, and background executions to **Temporal**.
|
||||
|
||||
This allows you to decouple workloads and run specialized worker nodes on separate, dedicated machines.
|
||||
|
||||
---
|
||||
|
||||
## How it Works
|
||||
|
||||
When configured for Temporal, the core Shumai web application does not process background jobs itself. Instead, it publishes tasks to a Temporal Cluster queue.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
UserUpload[User Uploads Video] --> WebApp[Shumai Web UI / API]
|
||||
WebApp -->|Register Task| Temporal[Temporal Cluster]
|
||||
Temporal -->|Queue: Transcoding| TranscodeWorker[Transcode Worker Machine]
|
||||
Temporal -->|Queue: AI Agents| AgentWorker[Agent Worker Machine]
|
||||
```
|
||||
|
||||
- **Transcoding Workers**: Can be run on GPU-accelerated or high-CPU compute instances.
|
||||
- **AI Agent Workers**: Can be run on isolated, sandboxed hosts to safely execute automation scripts.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Storage Requirement**: Because the core application and the background workers run on separate machines in a distributed setup, **you must configure an S3-compatible storage backend** (such as AWS S3, Cloudflare R2, or MinIO). Local storage (`STORAGE_BACKEND=local`) is not supported in this mode because the workers and the web application do not share a common filesystem.
|
||||
|
||||
---
|
||||
|
||||
## Step 1: Deploy the Core Stack
|
||||
|
||||
On your primary host machine, spin up the database, Temporal cluster services, and the main Shumai web application:
|
||||
|
||||
1. **Download the Temporal Stack Configuration**:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/temporal/docker-compose.yaml
|
||||
```
|
||||
2. **Configure S3 Settings**:
|
||||
Open the `docker-compose.yaml` file and set your S3 credentials and endpoint environment variables under the `shumai` service.
|
||||
3. **Start the Services**:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
This exposes the web application on port `3000`, the database on `5432`, the Temporal gRPC service on `7233`, and the Temporal management dashboard on `8080`.
|
||||
|
||||
---
|
||||
|
||||
## Step 2: Configure & Run Workers
|
||||
|
||||
On your specialized worker machines, you will run the dedicated background worker services. Each worker requires a `.env` file referencing the database, S3 storage, and the Temporal cluster:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@<primary-host-ip>:5432/shumai_db?schema=public
|
||||
WORKFLOW_EXECUTOR=temporal
|
||||
TEMPORAL_ADDRESS=<primary-host-ip>:7233
|
||||
STORAGE_BACKEND=s3
|
||||
AWS_ENDPOINT_URL_S3=https://your-s3-endpoint.com
|
||||
S3_BUCKET=shumai
|
||||
S3_REGION=us-east-1
|
||||
S3_ACCESS_KEY_ID=your-access-key-id
|
||||
S3_SECRET_ACCESS_KEY=your-secret-access-key
|
||||
```
|
||||
|
||||
### Media Transcoding Worker
|
||||
|
||||
Run this on your media processing nodes (requires system installations of `ffmpeg` and `ffprobe`):
|
||||
|
||||
```bash
|
||||
npm install -g @shumai-one/shumai-transcode
|
||||
shumai-transcode
|
||||
```
|
||||
|
||||
### AI Agent Worker
|
||||
|
||||
Run this on your sandboxed agent execution nodes (requires sandbox dependencies like `bubblewrap` or `sandbox-exec`):
|
||||
|
||||
```bash
|
||||
npm install -g @shumai-one/shumai-agent
|
||||
shumai-agent
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **For Bun Users:** Since the worker package binaries contain a `#!/usr/bin/env node` shebang, running `shumai-transcode` or `shumai-agent` directly will run under Node.js. If you wish to run them using Bun, prefix the commands with `bun run --bun` (e.g., `bun run --bun shumai-transcode` or `bun run --bun shumai-agent`).
|
||||
@@ -16,7 +16,43 @@
|
||||
"groups": [
|
||||
{
|
||||
"group": "Getting Started",
|
||||
"pages": ["introduction", "getting-started/docker"]
|
||||
"pages": ["getting-started/overview", "getting-started/quickstart"]
|
||||
},
|
||||
{
|
||||
"group": "Using Shumai",
|
||||
"pages": [
|
||||
"using-shumai/asset-management",
|
||||
"using-shumai/comments",
|
||||
"using-shumai/version-stack",
|
||||
"using-shumai/sort-and-search",
|
||||
"using-shumai/semantic-search",
|
||||
"using-shumai/team-project-management",
|
||||
"using-shumai/notifications",
|
||||
"using-shumai/metadata",
|
||||
"using-shumai/collections",
|
||||
"using-shumai/share-links"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Shumai Agent Settings",
|
||||
"pages": [
|
||||
"using-shumai/agent-providers",
|
||||
"using-shumai/agents",
|
||||
"using-shumai/agent-skills",
|
||||
"using-shumai/agent-sandbox"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Media Transcoding Settings",
|
||||
"pages": ["using-shumai/transcode-settings"]
|
||||
},
|
||||
{
|
||||
"group": "Configuration",
|
||||
"pages": ["configuration/env-variables"]
|
||||
},
|
||||
{
|
||||
"group": "Temporal Deployment",
|
||||
"pages": ["configuration/temporal"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
After Width: | Height: | Size: 87 KiB |
|
After Width: | Height: | Size: 94 KiB |
@@ -1,98 +0,0 @@
|
||||
---
|
||||
title: Get Started with Docker
|
||||
description: Learn how to spin up the Shumai stack using Docker Compose.
|
||||
---
|
||||
|
||||
We support two Docker Compose setups depending on your environment and use case:
|
||||
|
||||
1. **Local Pre-built Stack**: A lightweight setup to run the core API and frontend using pre-built images.
|
||||
2. **Temporal Stack**: A production-like environment with full background worker orchestration (Agent and Transcode workers) and Temporal services, built from your local source code.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, ensure you have the following installed on your machine:
|
||||
|
||||
- **Docker**: [Install Docker Desktop](https://docs.docker.com/get-docker/) or Docker Engine.
|
||||
- **Docker Compose**: Included in Docker Desktop, or [installed separately](https://docs.docker.com/compose/install/).
|
||||
- **Git**: To clone and manage the repository.
|
||||
|
||||
---
|
||||
|
||||
## 1. Local Pre-built Stack
|
||||
|
||||
This setup runs a PostgreSQL database (with `pgvector`) and the Shumai application container.
|
||||
|
||||
### Step 1: Navigate to the local docker-compose folder
|
||||
|
||||
From the root of your Shumai workspace, run:
|
||||
|
||||
```bash
|
||||
cd docker-compose/local
|
||||
```
|
||||
|
||||
### Step 2: Start the services
|
||||
|
||||
Spin up the containers in detached mode:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### Step 3: Access the application
|
||||
|
||||
Once all containers are running and healthy:
|
||||
|
||||
- Open your browser and navigate to **`http://localhost:3000`** to access the Shumai Web UI.
|
||||
|
||||
### Step 4: Tear down
|
||||
|
||||
To stop and remove the containers, run:
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Temporal Stack (Recommended for Development)
|
||||
|
||||
This setup builds the containers from your local workspace source code and starts the API, workers, and Temporal scheduler.
|
||||
|
||||
### Step 1: Navigate to the temporal docker-compose folder
|
||||
|
||||
From the root of your Shumai workspace, run:
|
||||
|
||||
```bash
|
||||
cd docker-compose/temporal
|
||||
```
|
||||
|
||||
### Step 2: Start the services
|
||||
|
||||
Build the images and run the full stack:
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
### Step 3: Access the services
|
||||
|
||||
- **Web UI**: Access the Shumai application at **`http://localhost:3000`**.
|
||||
- **Temporal Dashboard**: View active workflows and orchestration logs at **`http://localhost:8080`**.
|
||||
|
||||
### Step 4: Tear down
|
||||
|
||||
To stop the services and clean up volumes, run:
|
||||
|
||||
```bash
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Port Conflicts
|
||||
|
||||
If you receive port conflict errors, ensure that port `3000` (Web UI), `5432` (PostgreSQL), `7233` (Temporal), or `8080` (Temporal UI) are not currently being used by other processes on your machine.
|
||||
@@ -0,0 +1,37 @@
|
||||
---
|
||||
title: Overview
|
||||
description: Learn about Shumai, a modern collaboration and media review platform.
|
||||
---
|
||||
|
||||
## What is Shumai?
|
||||
|
||||
**Shumai** is a self-hosted media collaboration platform designed for creative teams, video editors, designers, and clients. It provides a centralized workspace where you can upload, organize, and review assets (video, audio, images, and documents) in real-time. With built-in AI capabilities and background transcoding, Shumai helps streamline asset organization and feedback workflows.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Core Features
|
||||
|
||||
- **Secure Storage**: Host your creative assets locally on your server or connect directly to any S3-compatible cloud storage provider (like AWS S3, Cloudflare R2, or MinIO).
|
||||
- **Media Commenting & Drawing**: Speed up review cycles with frame-level comments and drawing tools that let you sketch directly onto video frames or images.
|
||||
- **Version Control**: Stack multiple versions of the same file into a single entry to keep your workspace organized and track revision history.
|
||||
- **AI-Powered Tags & Search**: Automatically tag your files on upload using Google Gemini, and find assets based on visual or conceptual meaning.
|
||||
- **Custom Automations**: Build tailored background workflows using custom scripts to automate repetitive team actions.
|
||||
- **Proxy Transcoding**: Automatically convert heavy camera and design files in the background into web-friendly streaming formats for smooth playback.
|
||||
|
||||
---
|
||||
|
||||
## Deployment Scales
|
||||
|
||||
Depending on your team size and production volume, Shumai can be configured to run in one of two modes:
|
||||
|
||||
### 1. Single-Server Deployment (Default)
|
||||
|
||||
Ideal for small teams, local development, or low-volume workflows. In this setup, the main web server, database, and background tasks (like transcoding and AI tagging) all run on a single machine.
|
||||
|
||||
### 2. High-Scale Distributed Deployment
|
||||
|
||||
Recommended for production environments with high-volume video uploads. In this setup, background tasks are managed via a centralized queue (Temporal). This allows you to offload heavy workloads to specialized worker machines (for example, putting video transcoding tasks on high-CPU/GPU machines and AI agents on separate sandboxed environments).
|
||||
|
||||
_Note: S3-compatible cloud storage is required for distributed deployments so that all workers can access the uploaded files._
|
||||
@@ -0,0 +1,118 @@
|
||||
---
|
||||
title: Quickstart
|
||||
description: Get Shumai up and running using Docker, NPM, or from source.
|
||||
---
|
||||
|
||||
We support three installation options. For most administrators, **Docker Compose** is the fastest and easiest way to deploy Shumai.
|
||||
|
||||
---
|
||||
|
||||
## Option 1: Docker Compose (Recommended)
|
||||
|
||||
Docker Compose automatically configures and spins up all required services (the web application, a database, and local storage).
|
||||
|
||||
1. **Download the Configuration**:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.yaml
|
||||
```
|
||||
2. **Start the Services**:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
3. **Access the Dashboard**:
|
||||
Open your browser and navigate to **`http://localhost:3000`** to access the Shumai Web UI.
|
||||
|
||||
### Production Deployments (Temporal)
|
||||
|
||||
If you are deploying Shumai for a production team with high-volume video uploads, we recommend setting up a distributed deployment using **Temporal** and an S3-compatible storage backend. This offloads resource-heavy video transcoding and AI agents to separate, dedicated worker nodes to keep your main dashboard fast and responsive. For setup instructions, see the [Deploy with Temporal](/configuration/temporal) guide.
|
||||
|
||||
---
|
||||
|
||||
## Option 2: Install via NPM
|
||||
|
||||
You can run Shumai as a global service using the Node Package Manager (NPM).
|
||||
|
||||
### System Prerequisites
|
||||
|
||||
To handle media processing and security isolation, the host machine must have the following system utilities installed:
|
||||
|
||||
- **FFmpeg & FFprobe**: Required for video transcoding and metadata extraction.
|
||||
- _macOS_: `brew install ffmpeg`
|
||||
- _Linux (Ubuntu/Debian)_: `sudo apt-get install ffmpeg`
|
||||
- **Agent Sandbox Utilities**: Used to run custom automation scripts securely.
|
||||
- _macOS_: `brew install ripgrep`
|
||||
- _Linux (Ubuntu/Debian)_: `sudo apt-get install bubblewrap socat ripgrep`
|
||||
|
||||
### Step-by-Step Setup
|
||||
|
||||
1. **Start a Database**:
|
||||
Shumai requires a PostgreSQL database with the `pgvector` extension. Start a database container:
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
-e POSTGRES_USER=shumai \
|
||||
-e POSTGRES_PASSWORD=shumai_password \
|
||||
-e POSTGRES_DB=shumai_db \
|
||||
-p 5432:5432 \
|
||||
-d pgvector/pgvector:pg18
|
||||
```
|
||||
2. **Install Shumai**:
|
||||
```bash
|
||||
npm install -g @shumai-one/shumai
|
||||
```
|
||||
3. **Configure Environment Variables**:
|
||||
Create a new directory, navigate into it, and create a `.env` file:
|
||||
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
|
||||
Create a `.env` file with the following variables:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
BETTER_AUTH_SECRET=your-random-32-character-secret-key
|
||||
STORAGE_BACKEND=local
|
||||
AWS_ENDPOINT_URL_S3=http://localhost:3000
|
||||
```
|
||||
|
||||
> [!TIP]
|
||||
> `AWS_ENDPOINT_URL_S3` must include the scheme and port (e.g. `http://localhost:3000`) where Shumai is publicly accessible. This URL is used directly by client browsers for file uploads.
|
||||
|
||||
4. **Start Shumai**:
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **For Bun Users:** Since the published package binaries contain a `#!/usr/bin/env node` shebang, running `shumai` directly will execute under Node.js. If you installed using Bun and want to run it under the Bun runtime, prefix the command with `bun run --bun` (e.g., `bun run --bun shumai`).
|
||||
|
||||
```bash
|
||||
shumai
|
||||
```
|
||||
|
||||
Open your browser to `http://localhost:3000`.
|
||||
|
||||
---
|
||||
|
||||
## Option 3: Run from Source (For Developers)
|
||||
|
||||
If you wish to run Shumai locally to customize the platform or contribute:
|
||||
|
||||
1. **Clone the Repository**:
|
||||
```bash
|
||||
git clone https://github.com/shumaiOne/shumai.git
|
||||
cd shumai
|
||||
```
|
||||
2. **Install Dependencies**:
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
3. **Start a Database**:
|
||||
Launch a local database container (as described in the NPM section above).
|
||||
4. **Run Migrations**:
|
||||
Set up your `.env` file at the root, then apply database migrations:
|
||||
```bash
|
||||
bun run db:migrate
|
||||
```
|
||||
5. **Start the Dev Server**:
|
||||
```bash
|
||||
bun run dev
|
||||
```
|
||||
This spins up the web server on port `3000` with hot-reloading active.
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
title: Introduction
|
||||
description: Welcome to Shumai, a modern AI agent and media processing platform.
|
||||
---
|
||||
|
||||
## What is Shumai?
|
||||
|
||||
**Shumai** is a highly decoupled, modular platform built with a layered monorepo architecture. It integrates a React-based frontend, a fast Hono API backend, business logic services, and background workflow engines (supporting both local and Temporal-based execution).
|
||||
|
||||
### Key Features
|
||||
|
||||
- **Decoupled Monorepo Structure**: Managed via Bun Workspaces, ensuring each domain or layer is its own self-contained package.
|
||||
- **Robust API & Core Service Layers**: Built on Bun and Hono, featuring strict end-to-end type safety with Zod schemas and DTOs.
|
||||
- **Workflow & Activity Engine**: An orchestration engine capable of polling-based execution locally and enterprise-grade execution via Temporal.
|
||||
- **Modern UI**: A responsive, clean React interface powered by Tailwind CSS and Shadcn UI.
|
||||
- **Database Resilience**: Prisma ORM with PostgreSQL and `pgvector` for vector similarity and search capabilities.
|
||||
|
||||
## Explore the Documentation
|
||||
|
||||
Start by setting up Shumai on your local machine using our step-by-step guides.
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Get Started with Docker" icon="docker" href="/getting-started/docker">
|
||||
Learn how to build and spin up the Shumai stack quickly using Docker Compose.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 23 KiB |
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 40 KiB |
|
After Width: | Height: | Size: 52 KiB |
|
After Width: | Height: | Size: 309 KiB |
|
After Width: | Height: | Size: 65 KiB |
|
After Width: | Height: | Size: 70 KiB |
|
After Width: | Height: | Size: 45 KiB |
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 49 KiB |
|
After Width: | Height: | Size: 32 KiB |
@@ -0,0 +1,49 @@
|
||||
---
|
||||
title: AI Providers
|
||||
description: Connect Shumai to OpenAI, Anthropic, Gemini, or custom LLM gateways.
|
||||
---
|
||||
|
||||
When creating agents for chat or metadata autofill, you must select an underlying AI model. The available models and providers are configured globally within your team settings.
|
||||
|
||||
> 🔒 **Access Control:** Only **Team Owner** can manage AI providers. To update these settings, click your avatar, navigate to **Settings**, and select **Providers** from the left sidebar.
|
||||
|
||||
---
|
||||
|
||||
## Built-In Providers
|
||||
|
||||
Shumai ships with pre-configured configurations for major LLM providers. Because Shumai uses [Pi](https://github.com/earendil-works/pi) as its core agent framework, the supported provider and model configurations are automatically synchronized with Pi's capabilities.
|
||||
|
||||
Team Owner can freely edit, delete, or add new providers. All configured provider and model settings are injected directly into the Pi framework at runtime.
|
||||
|
||||
---
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Provider Settings
|
||||
|
||||
When configuring a provider, you will need to specify the following fields:
|
||||
|
||||
| Field | Description | Example / Notes |
|
||||
| :---------------- | :------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Provider Name** | A unique, human-readable label for the provider. | `Ollama-Local`, `Enterprise-Gateway` |
|
||||
| **Base URL** | The target API endpoint URL. | `http://localhost:11434/v1` |
|
||||
| **API Protocol** | The streaming and payload implementation format. | Most OpenAI-compatible gateways use `openai-completions`. |
|
||||
| **API Key** | The authentication token. Leave blank for unauthenticated or local setups. | Accepts a literal value (`sk-...`) or an environment variable name (`MY_API_KEY`) to load secrets securely from the host system. |
|
||||
|
||||
### Model Settings
|
||||
|
||||
Each provider can host multiple models. You can configure the following attributes per model:
|
||||
|
||||
| Field | Description | Example / Notes |
|
||||
| :-------------------- | :---------------------------------------------------------- | :------------------------------------------------- |
|
||||
| **Model ID** | The exact identifier string expected by the upstream API. | `gpt-4o`, `claude-3-5-sonnet-20240620` |
|
||||
| **Display Name** | An optional, friendly label shown to users in dropdowns. | `GPT-4 Omni`, `Claude 3.5` |
|
||||
| **Reasoning Support** | Toggle to enable extended thinking protocols. | Use for models with native reasoning capabilities. |
|
||||
| **Context Window** | The maximum total token capacity (input + output). | |
|
||||
| **Max Output Tokens** | The hard ceiling for generated response tokens per request. | |
|
||||
| **Input Cost** | Cost tracking specified per million input tokens. | Optional; used to calculate usage metrics. |
|
||||
| **Output Cost** | Cost tracking specified per million output tokens. | Optional; used to calculate usage metrics. |
|
||||
|
||||
---
|
||||
|
||||

|
||||
@@ -0,0 +1,45 @@
|
||||
---
|
||||
title: Agent Sandbox
|
||||
description: Secure execution environment and network domain restrictions for AI agent scripts.
|
||||
---
|
||||
|
||||
Running automated scripts poses security risks. Shumai resolves this by executing all agent skills inside a secure, isolated **Sandbox** environment.
|
||||
|
||||
---
|
||||
|
||||
## Sandbox Security
|
||||
|
||||
The sandbox restricts the execution process from accessing the host filesystem, system processes, and internal networks:
|
||||
|
||||
- **Isolation**: Scripts run in a virtualized workspace container that prevents them from modifying critical system files or reading other project data.
|
||||
- **Network Bridging**: External network calls from inside the sandbox are strictly monitored.
|
||||
|
||||
### File System Restrictions
|
||||
|
||||
To protect your workspace environments and sensitive data, the sandbox enforces strict file-level access controls:
|
||||
|
||||
- **Read Access**: The sandbox is allowed to read files within the workspace, with a strict exception for sensitive environment and certificate files. It cannot read: `['.env', '.env.*', '*.pem', '*.key']`.
|
||||
- **Write Access**: The sandbox is heavily restricted and may only write to the `.pi` folder located in the current working directory (cwd). Even within this allowed directory, the sandbox is explicitly blocked from writing or creating sensitive files matching: `['.env', '.env.*', '*.pem', '*.key']`.
|
||||
|
||||
---
|
||||
|
||||
## Domain Whitelisting
|
||||
|
||||
By default, sandboxed scripts cannot make network requests to the internet. Team Owners must explicitly authorize external APIs by managing domain access:
|
||||
|
||||
### Allowed Domains
|
||||
|
||||
A list of domains that sandboxed scripts are permitted to access.
|
||||
|
||||
- Default whitelisted domains include standard package registries (like npm or python package hosts) and repository hosts (like `github.com`).
|
||||
- If a script needs to call an external API (e.g. `api.slack.com`), the administrator must add it to the allowed list.
|
||||
|
||||
### Pending Domains
|
||||
|
||||
When an agent script attempts to reach a domain that is not currently authorized:
|
||||
|
||||
1. The network connection is blocked immediately.
|
||||
2. The target domain is logged and added to the **Pending Domains** list.
|
||||
3. Admins can review this list and click **Approve** to authorize the domain, or **Reject** to keep it blocked.
|
||||
|
||||

|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
title: Agent Skills
|
||||
description: Extend your agents' capabilities with custom automation tools.
|
||||
---
|
||||
|
||||
By default, AI models can only process text and image. In Shumai, you can extend both your **Chat Agents** and **Autofill Agents'** capabilities by registering **Skills**—custom scripts that the agent can run when solving a task.
|
||||
|
||||
---
|
||||
|
||||
## How Skills Work
|
||||
|
||||
Skills are actions that you define and make available to your AI agents.
|
||||
|
||||
- **Descriptions for the AI**: When you register a skill, the AI agent reads its metadata to understand exactly when and how it should use the tool to solve a problem.
|
||||
- **Execution & Local Caching**: When an agent decides it needs to use a skill, Shumai handles the execution. Newly added skill data is synced to the database first. Upon execution, the agent downloads and caches the skill locally from the database for fast, reliable access.
|
||||
|
||||
---
|
||||
|
||||
## Adding a Skill
|
||||
|
||||
To install a new skill into Shumai:
|
||||
|
||||
1. Navigate to the **Settings** page and open the **Skills** tab.
|
||||
2. Click the **Add Skill** button.
|
||||
3. In the dialog box, choose your installation method:
|
||||
- **Upload**: Select and upload a `.zip` file containing your skill.
|
||||
- **GitHub**: Provide a valid GitHub repository path.
|
||||
|
||||
> **Important:** Regardless of the installation method, the root folder of the skill must contain a `SKILL.md` file.
|
||||
|
||||
### Updating an Existing Skill
|
||||
|
||||
To update a skill, simply follow the steps above and install the new version using the exact same skill name. Shumai will automatically replace the older version.
|
||||
|
||||
---
|
||||
|
||||
## Configuring Environment Variables
|
||||
|
||||
If your custom skill requires environment variables (e.g., API keys, external tokens) to function:
|
||||
|
||||
1. Click on the installed **Skill Card** in the Skills tab to open the configuration dialog.
|
||||
2. Add the required environment variables.
|
||||
3. _(Optional)_ Set a default value for the variable. If no default value is provided, the skill will dynamically load the required value from the system environment at runtime.
|
||||
|
||||
---
|
||||
|
||||
## Enabling Skills for Agents
|
||||
|
||||
**All skills are disabled by default.** Merely installing a skill does not give agents access to it. To allow an agent to use an installed skill:
|
||||
|
||||
1. Navigate to your specific **Agent Settings**.
|
||||
2. Explicitly enable the skills you want to authorize for that particular agent.
|
||||
|
||||

|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
title: AI Agents
|
||||
description: Create and configure specialized AI agents for chat, metadata autofill, and vector embedding.
|
||||
---
|
||||
|
||||
Shumai uses specialized AI Agents to perform background automation, chat workflows, and semantic indexing. **Only Team Owner** can create, manage, and change agent settings.
|
||||
|
||||
---
|
||||
|
||||
## Agent Types
|
||||
|
||||
Shumai supports three types of agents:
|
||||
|
||||
### 1. Chat Agent
|
||||
|
||||
An interactive, conversational bot that appears in your project workspace.
|
||||
|
||||
- You can create as many chat agents as you want.
|
||||
- You can use "@" to mention chat agents directly in file comments.
|
||||
- The agent is context-aware: it can read files, search the project folder, inspect metadata, and read comment threads.
|
||||
- You can add "Skills" to a chat agent, enabling it to write files, run scripts, or interact with external APIs on your behalf.
|
||||
|
||||
### 2. Autofill Agent
|
||||
|
||||
A background agent that monitors file uploads. You can create **only one** Autofill Agent.
|
||||
|
||||
- Once created and enabled, Shumai automatically triggers the agent when a new file is uploaded, provided the file type is supported (image or video).
|
||||
- It analyzes the file's content (multimodal images, text descriptions) and populates any metadata fields flagged for "AI Autofill".
|
||||
|
||||
### 3. Embedding Agent
|
||||
|
||||
A specialized utility agent that processes assets to generate search indexes. You can create **only one** Embedding Agent.
|
||||
|
||||
- Once created and enabled, Shumai automatically runs embedding generation for new file uploads if the file type is supported (image or video).
|
||||
- These search indexes are saved to power Shumai's natural language semantic search capabilities.
|
||||
- **Note:** The Embedding Agent is hardcoded to use `google-embedding-2` as the LLM model. Because of this, you cannot choose a provider or model. Make sure you set the `GEMINI_API_KEY` environment variable to make it work.
|
||||
- **Chunk Duration:** You can configure the duration of chunks used for long video indexing via the `EMBEDDING_CHUNK_DURATION` environment variable (defaults to 60 seconds).
|
||||
|
||||
---
|
||||
|
||||
## Configuring an Agent
|
||||
|
||||
To edit or create an agent, navigate to **Team Settings** > **Agents** (Note: Only Team Owners can change these settings):
|
||||
|
||||
- **Avatar**: Choose an avatar for the agent by picking from the predefined avatars.
|
||||
- **Enabled / Disabled Toggle**: Stop or start the agent. If the embedding agent is disabled, semantic search indexing is paused.
|
||||
- **System Prompt / Soul**: The core instructions that define the agent's behavior, guidelines, formatting requirements, and operational limits.
|
||||
- **Model Provider**: Select which registered provider and model the agent should use for execution. _(Note: This option is disabled for Embedding Agents)._
|
||||
- **Assigned Skills**: Select which custom skills this agent is permitted to execute.
|
||||
|
||||

|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
title: Asset Management
|
||||
description: Learn how to upload, organize, delete, and recover assets in your workspace.
|
||||
---
|
||||
|
||||
Shumai makes it easy to bring creative media into your workspace, organize them in folders, and manage their lifecycle from upload to deletion.
|
||||
|
||||
---
|
||||
|
||||
## Uploading Assets
|
||||
|
||||
You can upload files and folders into Shumai using three methods:
|
||||
|
||||
1. **Drag and Drop**: Drag files or entire folder directories directly from your computer's file explorer and drop them anywhere on the Shumai dashboard interface.
|
||||
2. **Upload Buttons**: Click the **Upload File** or **Upload Folder** buttons in the toolbar.
|
||||
3. **Copy and Paste**: Copy an image or file from another website, application, or your clipboard, and paste it directly (`Ctrl+V` or `Cmd+V`) into the Shumai browser window to upload it instantly.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Folder Organization
|
||||
|
||||
When you upload an entire folder, Shumai replicates your folder hierarchy exactly inside the project.
|
||||
|
||||
- All subfolders are created automatically.
|
||||
- Nested files remain in their respective subfolders.
|
||||
- There is no limit to the depth of nested folders you can upload.
|
||||
|
||||
---
|
||||
|
||||
## Upload and Processing States
|
||||
|
||||
Once an upload starts, your assets progress through the following states:
|
||||
|
||||
- **Uploading**: The file is transferring to your storage server. The UI displays an active progress percentage.
|
||||
- **Processing**: The file has successfully uploaded. Shumai is now extracting media parameters (such as resolution, video length, and audio codecs) and creating a web-friendly streaming preview (proxy).
|
||||
- **Ready**: Processing is complete. The asset is fully visible, playable, and ready for comments, metadata tagging, or search.
|
||||
|
||||
---
|
||||
|
||||
## Deleting and Restoring Assets
|
||||
|
||||
If you no longer need an asset, you can delete it to clean up your workspace:
|
||||
|
||||
### The Trash Bin (Recently Deleted)
|
||||
|
||||
- When you delete a file or folder, it is moved to the **Recently Deleted** tab in your project settings.
|
||||
- Deleted assets are kept in the Trash for **30 days**. During this window, you can select any asset and click **Recover** to restore it to its original location.
|
||||
- If you want to free up space immediately, you can open the Recently Deleted tab and click **Empty Trash** to permanently purge the files.
|
||||
|
||||
### Permanent Removal
|
||||
|
||||
After **30 days** in the Trash, assets are automatically and permanently purged. Once purged, the records are removed from the database, and the files are deleted from your storage backend (local file system or S3 bucket) and cannot be recovered.
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
title: Collections
|
||||
description: Build dynamic virtual folders using metadata filters and search criteria.
|
||||
---
|
||||
|
||||
Instead of copying files into static folders, Shumai uses **Collections**—dynamic, query-based virtual views. A collection automatically groups files that match specific criteria in real-time.
|
||||
|
||||
---
|
||||
|
||||
## What is a Collection?
|
||||
|
||||
A **Collection** does not duplicate files on disk. Instead, it runs a pre-defined filter query against your project assets database. Whenever a new file is uploaded or metadata is updated to match the collection's query, the file automatically appears inside the collection.
|
||||
|
||||
---
|
||||
|
||||
## Creating a Collection
|
||||
|
||||
You can create collections in two ways:
|
||||
|
||||
- **From the Sidebar**: Click the `+` icon next to **Collections** in the project sidebar to open the query builder, configure your search rules, and save it.
|
||||
- **From the Search Dialog**: When searching and filtering files within the Search Dialog, you can save your active search criteria by clicking the **Save as collection** button at the bottom of the dialog. This will create a dynamic collection that mirrors your search filters.
|
||||
|
||||
---
|
||||
|
||||
## Setting Up Collection Filters
|
||||
|
||||
When creating or editing a collection, you specify a name and set up a filter query in the builder interface:
|
||||
|
||||
### 1. Source Folder
|
||||
|
||||
Specify the starting folder directory. The collection will scan this folder and all its nested subfolders recursively.
|
||||
|
||||
### 2. Operators and Conditions
|
||||
|
||||
- **Logical Operators**: Combine conditions using **AND** (all conditions must be met) or **OR** (at least one condition must be met).
|
||||
- **Fields**: Filter on standard fields (name, type, size) or custom metadata fields (such as "Status" or "Approval").
|
||||
- **Comparison Rules**:
|
||||
- _Equals / Does Not Equal_: Match exact values.
|
||||
- _Contains / Does Not Contain_: Match parts of words.
|
||||
- _Greater Than / Less Than_: Set up numeric or date ranges (e.g., date uploaded is after yesterday).
|
||||
- _Is Empty / Is Not Empty_: Check if fields have values.
|
||||
|
||||
### 3. Collection Default Sorting
|
||||
|
||||
Define a custom sorting rule specific to the collection (e.g., sort by the custom field "Publication Date" ascending) to override the default project sort order.
|
||||
|
||||
---
|
||||
|
||||
## Example Use Cases
|
||||
|
||||
- **Review Dashboard**: A collection named "Pending Review" that displays assets where `Status` equals `In Review`.
|
||||
- **Daily Deliverables**: A collection filtering files uploaded in the last 24 hours.
|
||||
- **Approved Assets**: A collection filtering assets matching `Tag` contains `marketing` AND `Approved` is `Yes`.
|
||||
|
||||

|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
title: Comments & Annotations
|
||||
description: Collaborate on media assets using drawings and timestamped comments.
|
||||
---
|
||||
|
||||
Shumai provides a review workspace designed for video, audio, and image feedback. Instead of generic text comments, you can drop timestamped feedback and sketch directly onto the media.
|
||||
|
||||
---
|
||||
|
||||
## Timestamped Comments
|
||||
|
||||
When reviewing video or audio files, comments can be anchored to specific seconds on the timeline:
|
||||
|
||||
1. Play or scrub the video player to the exact frame or timestamp you want to review.
|
||||
2. Type your feedback in the comment box.
|
||||
3. Click **Submit**. Your comment will be saved with a timestamp marker shown on the video timeline.
|
||||
4. Collaborators can click your comment card in the sidebar to jump the video player to the exact moment you commented on.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Draw Annotations (Canvas)
|
||||
|
||||
To give clear visual feedback, you can draw directly onto image or video frames:
|
||||
|
||||
1. Pause the video on the frame you wish to annotate.
|
||||
2. Select a drawing tool (brush, rectangle, circle, arrow) from the annotation toolbar overlaying the player.
|
||||
3. Draw directly on the video screen.
|
||||
4. Type your comment and click **Submit**. Your drawing is saved along with your comment.
|
||||
5. When other users select your comment, your drawings will overlay on the player screen automatically at that frame.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Comment Threads (Replies)
|
||||
|
||||
Keep discussions organized by replying directly to feedback:
|
||||
|
||||
- Click **Reply** on any comment.
|
||||
- Replies are nested under the parent comment card, making it easy to track resolution conversations.
|
||||
|
||||
---
|
||||
|
||||
## Attachments
|
||||
|
||||
You can attach reference documents, style guides, or other assets to your comments:
|
||||
|
||||
- Click the **Paperclip** icon inside the comment input box.
|
||||
- Choose a file to upload. It will appear as a downloadable link directly on your comment card.
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
title: Asset Metadata
|
||||
description: Customize your assets with dynamic, scoped metadata fields and AI-driven autofill features.
|
||||
---
|
||||
|
||||
Standard file systems limit you to basic attributes like file name and modification date. Shumai lets you create structured metadata fields tailored to your creative pipeline—from tracking review statuses to capturing technical specs automatically.
|
||||
|
||||
---
|
||||
|
||||
## Metadata Scopes
|
||||
|
||||
Fields are grouped and displayed in the sidebar inspector by their scope:
|
||||
|
||||
- **Essentials (System Scope)**: Core fields built into Shumai that appear across all teams and projects. Many of these (like resolution, duration, and codec) are automatically extracted during file upload.
|
||||
- **Team Attributes (Team Scope)**: Custom fields defined at the team level, which are inherited by all projects under that team. Excellent for enforcing company-wide metadata guidelines.
|
||||
- **Custom Fields (Project Scope)**: Unique metadata fields defined specifically for a single project workspace.
|
||||
|
||||
---
|
||||
|
||||
## Custom Field Types
|
||||
|
||||
When building or editing your metadata schema, you can choose from these custom types:
|
||||
|
||||
- **Text**: A single-line input field for simple strings, codes, or labels.
|
||||
- **Long Text**: A paragraph input area for detailed descriptions, summaries, or notes.
|
||||
- **Dropdown (Select)**: A single-choice dropdown menu where options can be styled with custom background colors (e.g., Green for Approved, Yellow for In Progress).
|
||||
- **Tags (Multi-Select)**: A multi-choice tag field allowing you to assign multiple custom colored labels to a single asset.
|
||||
- **Star Rating**: An interactive star rating scale (supports configurations from 2 to 6 stars) for scoring and ranking.
|
||||
- **Number**: A numeric field supporting custom decimal precision (from 0 to 3 decimal places) for values like frame rates, counts, or scores.
|
||||
- **Checkbox (Toggle)**: A simple Yes/No switch for binary indicators (e.g., "VFX Required").
|
||||
- **Date**: A calendar date selector with support for custom display formats (Local, ISO, USA, Euro, Friendly) and optional time tracking.
|
||||
- **Single User**: A dropdown select field populated dynamically with team members, displaying their name and avatar.
|
||||
- **Multiple User**: A multi-choice user select field populated dynamically with team members, allowing you to assign multiple team members.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Default Built-in Fields
|
||||
|
||||
Every project comes pre-configured with several essential fields populated automatically during upload or status changes:
|
||||
|
||||
- **Technical Specs**: File Size, Format, File Type (Video, Audio, Image, Document, File), Resolution Width/Height, Video/Audio Codecs, Duration, Frame Rate, and Audio Sample Rates.
|
||||
- **Collaborative Fields**:
|
||||
- **Status**: A dropdown select field containing `Needs Review` (Red), `In Progress` (Yellow), and `Approved` (Green).
|
||||
- **Rating**: A 5-star rating scale for quick review evaluations.
|
||||
- **Keywords**: A tags (multi-select) field to categorize assets.
|
||||
|
||||
---
|
||||
|
||||
## Field Settings & Controls
|
||||
|
||||
### Read-Only Fields
|
||||
|
||||
Marking a custom field as **Read-Only** prevents team members from editing its value directly. This is ideal for fields meant to be populated programmatically or by automated integrations.
|
||||
|
||||
### AI Autofill
|
||||
|
||||
When you enable **AI Autofill** on a metadata field, Shumai automatically analyzes uploaded assets using your team's default Gemini AI model. The AI will populate the field with extracted insights:
|
||||
|
||||
- For text/long text: AI generates summaries, descriptions, or shot logs.
|
||||
- For select/multi-select: AI automatically categorizes themes or applies matching tags.
|
||||
- For numbers: AI can output confidence scores or other metrics.
|
||||
|
||||
### Visibility & Ordering
|
||||
|
||||
In the metadata inspector:
|
||||
|
||||
- **Visibility**: Toggle fields on or off to clean up the sidebar and only show what is relevant to your current review process.
|
||||
- **Ordering**: Drag and drop fields in the builder to arrange their display order within the Custom Fields section.
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
title: Notifications
|
||||
description: Stay updated on comment activity, uploads, and project updates.
|
||||
---
|
||||
|
||||
Shumai includes a built-in notification system to keep you informed of team activities, file statuses, and review comments. You can fully customize which events trigger notifications, check updates from your inbox, or toggle notifications for specific projects.
|
||||
|
||||
---
|
||||
|
||||
## Notification Settings
|
||||
|
||||
You can customize your notification preferences for each team to avoid unnecessary noise. To manage your notification events:
|
||||
|
||||
1. Click on **Team Settings** in the sidebar.
|
||||
2. Select the **Notifications** tab.
|
||||
3. Toggle the switches to enable or disable notifications for specific events:
|
||||
|
||||
### Comments & Replies
|
||||
|
||||
- **General Comments**: Notify me when someone comments on an asset I collaborate on.
|
||||
- **Comment Replies**: Notify me when someone replies to my comment threads.
|
||||
- **@Mentions**: Notify me when someone tags me (using `@username`) in any comment.
|
||||
|
||||
### Assets & Statuses
|
||||
|
||||
- **Your Uploads**: Notify me when files I upload finish transcoding and are ready for review.
|
||||
- **Other Uploads**: Notify me when other team members upload new assets.
|
||||
- **Status Updates**: Notify me when someone changes an asset's status (e.g., from "In Progress" to "Approved").
|
||||
|
||||
---
|
||||
|
||||
## Project-Level Notifications
|
||||
|
||||
If you want to mute a specific project entirely (except for direct `@mentions` which are always delivered):
|
||||
|
||||
1. Navigate to the **Project** dashboard.
|
||||
2. Open **Project Settings**.
|
||||
3. Toggle **Enable Notifications** on or off.
|
||||
|
||||
---
|
||||
|
||||
## The Notification Inbox
|
||||
|
||||
- The **Bell** icon in the navigation bar displays a red badge when you have unread notifications.
|
||||
- Click the bell to open your notification feed panel.
|
||||
- Click any notification card to mark it as read and jump directly to the relevant file and comment thread.
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
title: Semantic Search
|
||||
description: Find files based on visual and textual meaning using AI vector similarity.
|
||||
---
|
||||
|
||||
Traditional search relies on matching exact characters (e.g., searching "dog" will not find files named "golden_retriever.jpg"). Shumai resolves this with **Semantic Search**, which indexes the conceptual meaning of your files.
|
||||
|
||||
---
|
||||
|
||||
## How Semantic Search Works
|
||||
|
||||
Semantic search translates text queries and file contents into conceptual profiles. Similar ideas and visuals are mapped near each other.
|
||||
|
||||
1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile.
|
||||
- **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing.
|
||||
- **Chunking**: For long videos, Shumai breaks them into chunks to ensure high-quality indexing. By default, chunks are **60 seconds** long with a **5-second overlap** to ensure no context is lost at the boundaries.
|
||||
- **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames.
|
||||
- **Optimization**: To optimize processing, Shumai automatically transcodes each chunk to **1 FPS** and **removes the audio track** before sending it to the embedding agent.
|
||||
- **Configuration**: Administrators can customize these settings using the `EMBEDDING_CHUNK_DURATION` and `EMBEDDING_CHUNK_OVERLAP` environment variables.
|
||||
2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets.
|
||||
3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`.
|
||||
|
||||
### Understanding Search Results
|
||||
|
||||
Semantic search works differently than traditional keyword filtering:
|
||||
|
||||
- **Sort vs. Filter**: Semantic search acts more like a **sort** than a strict filter. While the most relevant results will appear at the top, less related content may still be present further down the list, ranked lower. If you need to exclude specific types of files, use the **Filter** panel in conjunction with semantic search.
|
||||
- **Video Chunks**: Because Shumai indexes videos in chunks, a single long video may appear multiple times in your search results. Each result corresponds to a specific segment of the video where the match was found.
|
||||
- **Match Timestamps**: When a search result comes from a video chunk, Shumai displays the matching start and end time. Clicking on these results will open the video and automatically jump to the relevant timestamp.
|
||||
|
||||
---
|
||||
|
||||
## Activating Semantic Search
|
||||
|
||||
To use semantic search:
|
||||
|
||||
1. Verify that your administrator has enabled the **Embedding Agent** under **Team Settings** > **Agents**.
|
||||
2. Once active, Shumai will automatically begin indexing new uploads.
|
||||
3. Toggle the **Semantic** button next to the search input in the file explorer to search by conceptual meaning.
|
||||
|
||||

|
||||
@@ -0,0 +1,57 @@
|
||||
---
|
||||
title: Share Links
|
||||
description: Share files and folders securely with external clients and stakeholders.
|
||||
---
|
||||
|
||||
Shumai allows you to collaborate with external stakeholders who do not have accounts in your workspace. Using **Share Links**, you can grant view and comment access to specific folders or assets.
|
||||
|
||||
---
|
||||
|
||||
## Creating a Share Link
|
||||
|
||||
1. Select the files or folders you want to share, then right-click and choose **Create New Share Link**.
|
||||
2. Click the newly created **Share Link** in the project sidebar to open its settings page.
|
||||
3. Configure the share link in the right sidebar. You can:
|
||||
- Add password protection
|
||||
- Set an expiration date
|
||||
- Configure which metadata fields are visible to recipients
|
||||
4. Click **Copy Link** at the bottom of the sidebar, then send the link to your client or stakeholder.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Adding Assets to an Existing Share Link
|
||||
|
||||
Once a share link is created, you can easily add more files or folders to it without generating a new link:
|
||||
|
||||
1. Select the files or folders you want to add from your file browser.
|
||||
2. Drag and drop them directly onto the share link item in the project sidebar.
|
||||
3. The share link will immediately update to include the newly added assets.
|
||||
|
||||
---
|
||||
|
||||
## Security and Settings
|
||||
|
||||
### Expiration Date
|
||||
|
||||
Set an optional date and time after which the link becomes inactive. Once the date passes, the link returns a `404 Not Found` page to anyone attempting to access it.
|
||||
|
||||
### Password Protection
|
||||
|
||||
Secure sensitive work by adding a password. External users visiting the link will be prompted to enter the correct password before they can view the assets.
|
||||
|
||||
### Default Sort Order
|
||||
|
||||
Set a default sorting rule for the shared page (e.g. sorting by name or version) to ensure the client views the files in the intended order.
|
||||
|
||||
### Column Visibility
|
||||
|
||||
To protect proprietary information, you can configure exactly which metadata columns are visible to external users. For example:
|
||||
|
||||
- **Visible**: `Asset Name`, `Resolution`, `Download Link`.
|
||||
- **Hidden**: `Internal Budget Code`, `Assignee Notes`, `Cost`.
|
||||
|
||||
### Disabling a Link
|
||||
|
||||
If you need to revoke access immediately, navigate to the **Share Links** tab under Project settings and toggle the **Disabled** switch. You can re-enable the link at any time.
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
title: Sort & Search
|
||||
description: Find assets using the search dialog and organize them with custom sort orders.
|
||||
---
|
||||
|
||||
As your projects grow, finding the right media files quickly is critical. Shumai provides a powerful search dialog and flexible sorting controls.
|
||||
|
||||
---
|
||||
|
||||
## Searching Assets
|
||||
|
||||
Searching is performed inside a dedicated **Search Dialog** that you can open by clicking the search icon in the toolbar.
|
||||
|
||||

|
||||
|
||||
### Search Dialog Features
|
||||
|
||||
- **Keyword Matches**: Type a keyword in the text input to instantly filter file and folder names.
|
||||
- **Advanced Filtering**: Click to expand the filters panel and build multi-conditional queries (e.g. searching for files where the custom field "Status" equals "In Progress" AND the file type is "Video").
|
||||
- **Semantic Search Toggle**: If enabled by your team administrator, toggle "Semantic" on to search by conceptual meaning (e.g. searching "sunny beach" to find clips of oceans even if the filenames do not contain those words).
|
||||
- **Save as Collection**: Easily convert your active search criteria into a dynamic folder. Click **Save as Collection** inside the dialog to create the collection and navigate to it instantly.
|
||||
- **Infinite Scroll Results**: Search results load dynamically in real-time as you type or adjust filters.
|
||||
|
||||
---
|
||||
|
||||
## Sorting and Reordering Assets
|
||||
|
||||
You can change the sorting order of the asset list using the sort dropdown or by clicking metadata column headers:
|
||||
|
||||
- **Custom Sort (Default)**: By default, the project file explorer is set to **Custom** sorting. In this mode, you can manually drag and drop files and folders to reorder them exactly as you wish.
|
||||
- **Standard Sort Options**: You can also sort files by:
|
||||
- _Newest Upload_: Displays recently uploaded assets first.
|
||||
- _Name_: Sorts alphabetically (A-Z or Z-A).
|
||||
- _Size_: Sorts by file size.
|
||||
- _Type_: Groups folders together at the top, followed by version stacks and individual files.
|
||||
- _Custom Columns_: Sorts by dynamic metadata fields (e.g., sorting by "Due Date" or "Priority").
|
||||
|
||||
> [!NOTE]
|
||||
> **Drag and Drop Reordering Limit**: Drag-and-drop manual reordering is **only** available when the sort order is set to **Custom**. If you select any other sort option (such as Name or Size), manual reordering is disabled.
|
||||
@@ -0,0 +1,58 @@
|
||||
---
|
||||
title: Team & Project Management
|
||||
description: Organize your workspaces, define roles, and manage members.
|
||||
---
|
||||
|
||||
Shumai uses a two-tier organizational hierarchy—**Teams** and **Projects**—to manage file access and collaboration.
|
||||
|
||||
---
|
||||
|
||||
## Member Visibility & Scopes
|
||||
|
||||
Members added to a team belong to one of two permission scopes:
|
||||
|
||||
- **Team Scope**: Team-scoped members can view all projects inside the team by default.
|
||||
- **Project Scope**: Project-scoped members can only see and access the specific projects they have been explicitly granted permission to join.
|
||||
|
||||
---
|
||||
|
||||
## Member Roles
|
||||
|
||||
Shumai uses permission roles to control what actions team and project members can perform:
|
||||
|
||||
- **Owner**: Full administrative control. Team Owners can modify team settings, skills, and AI providers. Project Owners can edit project configurations, manage members, and generate invites.
|
||||
- **Editor**: Can upload files, edit custom metadata, create folders, and add comments or canvas annotations.
|
||||
- **Reviewer**: Read-only access. Can view media assets, toggle versions, and leave text comments or canvas drawings, but cannot upload/delete files or modify metadata.
|
||||
|
||||
---
|
||||
|
||||
## Project Role Overrides
|
||||
|
||||
Administrators can add existing team members to specific projects to override their permissions:
|
||||
|
||||
- Project-level roles take higher priority than team-level roles within that project.
|
||||
- _Example_: If Peter is a team member with an **Editor** role, an administrator can add Peter to _Project A_ with a **Reviewer** role. Peter will act as a Reviewer inside Project A, but maintain his Editor role across the rest of the team's projects.
|
||||
|
||||
---
|
||||
|
||||
## Managing Members
|
||||
|
||||
Member management is handled using a unified **Members Dialog** popup:
|
||||
|
||||
### 1. Managing Team Members
|
||||
|
||||
- **Access**: Open your **Team Dashboard** and click the members list or avatar group in the header.
|
||||
- **Settings**:
|
||||
- Change member roles between Editor and Reviewer.
|
||||
- Remove members from the team.
|
||||
- **Invite New Members**: Select a role (Editor or Reviewer) and click **Generate Link**. Copy the signup URL to invite new users to the team.
|
||||
|
||||
### 2. Managing Project Members
|
||||
|
||||
- **Access**: Open your project **File Browser** and click the members avatars or button in the top toolbar.
|
||||
- **Settings**:
|
||||
- Change project member roles or remove them from the project.
|
||||
- **Add Team Members**: The dialog displays a list of active team members who are not yet added to this project. Select their project role (which will override their team role inside this project) and click **Add** to add them instantly.
|
||||
- **Invite External Collaborators**: Select a role (Editor or Reviewer) and click **Generate Link** to invite new users directly into this specific project.
|
||||
|
||||

|
||||
@@ -0,0 +1,36 @@
|
||||
---
|
||||
title: Media Transcoding Settings
|
||||
description: Configure video transcode strategies and preview proxy generation.
|
||||
---
|
||||
|
||||
High-resolution camera formats and raw media files cannot be rendered natively in web browsers. Shumai transcodes these assets into web-friendly streaming formats in the background.
|
||||
|
||||
Regardless of the selected transcoding strategy, Shumai's pipeline will **always** generate at least one lightweight `180p` proxy version of the raw file to ensure fast scrubbing, thumbnail generation, and instant previews.
|
||||
|
||||
---
|
||||
|
||||
## Video Transcode Strategies
|
||||
|
||||
Team Owners can select how the transcoding engine handles uploaded files. Shumai evaluates the original file's dimensions against its standard target resolutions (2160p, 1080p, 720p, 540p, and 360p) to determine the output.
|
||||
|
||||
### 1. Best Match
|
||||
|
||||
- **Behavior**: Shumai checks the longest dimension (width or height) of the uploaded file. It then finds the highest standard resolution that is equal to or smaller than the original file, preventing unnecessary upscaling. If the original video is exceptionally small, it defaults to a minimum target of `360p`.
|
||||
- **Output**: Generates exactly two web-optimized files: the universal `180p` proxy and the single calculated "best match" resolution.
|
||||
- **Use Case**: This is the recommended option. It ensures web compatibility without wasting processing time and storage space on redundant lower-resolution versions.
|
||||
|
||||
### 2. Transcode All
|
||||
|
||||
- **Behavior**: Shumai calculates the "best match" resolution exactly as it does in the previous strategy. However, instead of stopping at one version, it processes the video into the best match _and_ every standard resolution smaller than it.
|
||||
- **Output**: Generates a full cascade of files. For example, if the best match for an upload is `1080p`, Shumai will generate the `180p` proxy, `1080p`, `720p`, `540p`, and `360p` versions.
|
||||
- **Use Case**: Ideal for environments requiring adaptive bitrate streaming, ensuring consistent playback performance across all client devices regardless of varying network conditions.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Processing Architecture
|
||||
|
||||
Transcoding is resource-intensive. Depending on your scaling needs, your administrator can run the transcode engine on the main server (Single-Server mode) or delegate tasks to specialized transcoding worker machines (Temporal Worker mode).
|
||||
|
||||
Note: In all modes, the processing machine must have the standard system media utilities (`ffmpeg` and `ffprobe`) installed.
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
title: File Version Stack
|
||||
description: Group multiple revisions of an asset under a single version-controlled stack.
|
||||
---
|
||||
|
||||
When collaborating on creative assets, files naturally go through multiple iterations (e.g. `draft_v1.mp4`, `draft_v2.mp4`, `draft_final.mp4`). To prevent workspace clutter, Shumai allows you to bundle these revisions into a **Version Stack**.
|
||||
|
||||
---
|
||||
|
||||
## What is a Version Stack?
|
||||
|
||||
A **Version Stack** groups multiple revisions of the same file into a single entry. Instead of cluttering your folder with several similar files, Shumai collapses them into one item with a version badge (e.g., "V3") and a dropdown list of older versions.
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Creating & Adding to a Version Stack
|
||||
|
||||
You can create a version stack or add new versions to an existing stack using two interactive methods:
|
||||
|
||||
- **Drag and Drop**:
|
||||
- _To create a stack_: Drag one standalone file directly over another standalone file in your file explorer.
|
||||
- _To add a version_: Drag a standalone file directly onto an existing version stack.
|
||||
- **Context Menu (Right-Click)**:
|
||||
- Right-click any standalone file or existing version stack.
|
||||
- Choose **Create New Version** from the menu.
|
||||
- Select the revision file from your computer to upload it directly into the stack.
|
||||
|
||||
---
|
||||
|
||||
## Managing Versions
|
||||
|
||||
### Active Version
|
||||
|
||||
The file at the top of the stack is the **Active Version**. This is the version displayed on your project dashboard, search results, and metadata panels.
|
||||
|
||||
### Reordering and Reverting Versions
|
||||
|
||||
You can promote a past version or reorder your stack:
|
||||
|
||||
- Open the version stack details sidebar.
|
||||
- Drag and drop the versions in the list to change their order. The version you drag to the top automatically becomes the active version.
|
||||
- To remove a file from the stack, drag it out or select **Unstack**. This moves the file back to the parent folder as a standalone file.
|
||||
|
After Width: | Height: | Size: 83 KiB |
@@ -75,6 +75,7 @@ export default tseslint.config(
|
||||
'**/packages/core/src/generated/**',
|
||||
'**/data/**',
|
||||
'**/packages/webui/components/ui/**',
|
||||
'**/packages/webui/paraglide/**',
|
||||
],
|
||||
},
|
||||
js.configs.recommended,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.0.2",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
@@ -13,12 +13,15 @@
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "bun run --hot apps/web/src/index.ts",
|
||||
"i18n:compile": "paraglide-js compile --project ./packages/webui/project.inlang --outdir ./packages/webui/paraglide --emit-ts-declarations --no-emit-git-ignore",
|
||||
"i18n:watch": "paraglide-js compile --project ./packages/webui/project.inlang --outdir ./packages/webui/paraglide --watch --emit-ts-declarations --no-emit-git-ignore",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
||||
"test": "bun --bun vitest run",
|
||||
"format": "prettier --write .",
|
||||
"db:migrate": "bun run prisma migrate dev",
|
||||
"db:deploy": "bun run prisma migrate deploy",
|
||||
"db:generate": "cd packages/db && bunx prisma generate",
|
||||
"generate-providers": "bun run scripts/generate-builtin-providers.ts",
|
||||
"worker:agent": "bun apps/agent/src/index.ts",
|
||||
"worker:transcode": "bun apps/transcode/src/index.ts",
|
||||
@@ -34,24 +37,25 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "10.0.1",
|
||||
"@inlang/paraglide-js": "2.20.0",
|
||||
"@testcontainers/postgresql": "12.0.1",
|
||||
"@types/bun": "1.3.14",
|
||||
"@types/node": "25.9.1",
|
||||
"bun-plugin-pino": "1.5.0",
|
||||
"bun-plugin-tailwind": "0.1.2",
|
||||
"bun-types": "1.3.14",
|
||||
"pino-pretty": "13.1.3",
|
||||
"eslint": "10.4.1",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-plugin-prettier": "5.5.6",
|
||||
"eslint-plugin-react": "7.37.5",
|
||||
"globals": "17.6.0",
|
||||
"pino-pretty": "13.1.3",
|
||||
"prisma": "7.8.0",
|
||||
"testcontainers": "12.0.1",
|
||||
"typescript": "6.0.3",
|
||||
"typescript-eslint": "8.60.1",
|
||||
"vitest": "4.1.8",
|
||||
"vite-tsconfig-paths": "6.1.1"
|
||||
"vite-tsconfig-paths": "6.1.1",
|
||||
"vitest": "4.1.8"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tanstack/react-virtual": "3.14.2",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.0.2",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -127,6 +127,54 @@ describe('Agent Activities', () => {
|
||||
expect(res.text).toBe('Error: API key not valid')
|
||||
})
|
||||
|
||||
it('should pass thinkingLevel from agent config to createAgentSession', async () => {
|
||||
const mockHarness = {
|
||||
prompt: vi.fn().mockResolvedValue({
|
||||
content: [],
|
||||
usage: { input: 0, output: 0 },
|
||||
}),
|
||||
getThinkingLevel: vi.fn().mockReturnValue('high'),
|
||||
}
|
||||
const mockSession = {
|
||||
getEntries: vi.fn().mockResolvedValue([]),
|
||||
getStorage: vi.fn().mockReturnValue({ sessionId: 'mock-session-id' }),
|
||||
}
|
||||
|
||||
vi.mocked(piAgent.createAgentSession).mockResolvedValue({
|
||||
session: mockSession as unknown as Session<DatabaseSessionMetadata>,
|
||||
harness: mockHarness as unknown as AgentHarness,
|
||||
})
|
||||
|
||||
const context = {
|
||||
agent: {
|
||||
id: 'b1',
|
||||
provider: { name: 'google' },
|
||||
modelRef: { modelId: 'gemini' },
|
||||
config: { thinkingLevel: 'high' },
|
||||
},
|
||||
dbProviders: [],
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
} as unknown as AgentExecutionContext
|
||||
|
||||
await agentChatActivity({
|
||||
teamId: 't1',
|
||||
agentId: 'b1',
|
||||
message: 'Hi',
|
||||
imageUrls: [],
|
||||
projectId: 'p1',
|
||||
folderId: 'f1',
|
||||
sessionId: 'mock-session-id',
|
||||
context,
|
||||
})
|
||||
|
||||
expect(piAgent.createAgentSession).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
thinkingLevel: 'high',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should call autofillAiActivity and run autofill tool', async () => {
|
||||
const mockHarness = {
|
||||
prompt: vi.fn().mockResolvedValue({
|
||||
@@ -306,7 +354,7 @@ describe('Agent Database Activities Integration', () => {
|
||||
teamId: team.id,
|
||||
name: 'openai',
|
||||
config: {
|
||||
api: 'openai',
|
||||
api: 'openai-completions',
|
||||
apiKey: 'mock-key',
|
||||
},
|
||||
},
|
||||
|
||||
@@ -97,11 +97,16 @@ If you need to create files in the local filesystem (for example, a temporary fi
|
||||
systemPrompt += `\n\nYour current model supports the following input types: ${modelConfig.input.join(', ')}.`
|
||||
}
|
||||
|
||||
const agentConfig = agent.config as PrismaJson.AgentConfig | null | undefined
|
||||
const thinkingLevel = agentConfig?.thinkingLevel || 'off'
|
||||
|
||||
const { session, harness } = await createAgentSession({
|
||||
teamId: params.teamId,
|
||||
agentId: params.agentId,
|
||||
providerName,
|
||||
modelId,
|
||||
systemPrompt,
|
||||
thinkingLevel,
|
||||
teamSkills: teamSkills.map((s) => ({
|
||||
id: s.id,
|
||||
name: s.name,
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import {
|
||||
generateEmbeddingActivity,
|
||||
generateTextEmbeddingActivity,
|
||||
extractAiMetadataActivity,
|
||||
getEmbeddingContextActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
} from './ai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/transcode'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { prisma, AssetType, AssetStatus } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import * as fs from 'fs'
|
||||
|
||||
vi.mock('fs', async (importOriginal) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- importOriginal returns the raw fs module, typed dynamically as any here
|
||||
@@ -38,7 +38,7 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/transcode', () => ({
|
||||
vi.mock('@shumai/core/src/transcode/transcode', () => ({
|
||||
transcodeService: {
|
||||
extractVideoFrames: vi.fn(),
|
||||
},
|
||||
@@ -93,19 +93,13 @@ describe('AI Activities Unit Tests', () => {
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const context = {
|
||||
agent: { config: { provider: 'google', model: 'gemini' } },
|
||||
asset: { id: 'a1', mediaType: 'image/png', storageKey: { key: 'test.png' } },
|
||||
}
|
||||
|
||||
const res = await generateEmbeddingActivity({
|
||||
const res = await generateImageEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetId: 'a1',
|
||||
context,
|
||||
assetKey: 'test.png',
|
||||
mediaType: 'image/png',
|
||||
})
|
||||
|
||||
expect(res.embeddings.length).toBe(1)
|
||||
expect(res.embeddings[0].embedding).toEqual([0.1, 0.2, 0.3])
|
||||
expect(res.embedding).toEqual([0.1, 0.2, 0.3])
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'test.png')
|
||||
})
|
||||
|
||||
@@ -115,25 +109,16 @@ describe('AI Activities Unit Tests', () => {
|
||||
contentType: 'video/mp4',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const context = {
|
||||
agent: { config: { provider: 'google', model: 'gemini' } },
|
||||
asset: { id: 'a2', mediaType: 'video/mp4', storageKey: { key: 'test.mp4' } },
|
||||
}
|
||||
|
||||
const res = await generateEmbeddingActivity({
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetId: 'a2',
|
||||
context,
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
|
||||
})
|
||||
|
||||
// Duration is 120s, chunks are 60s, so it should generate 2 chunks (0-60, 60-120)
|
||||
expect(res.embeddings.length).toBe(2)
|
||||
expect(res.embeddings[0]).toEqual({ embedding: [0.1, 0.2, 0.3], startTime: 0, endTime: 60 })
|
||||
expect(res.embeddings[1]).toEqual({ embedding: [0.1, 0.2, 0.3], startTime: 60, endTime: 120 })
|
||||
|
||||
expect(vi.mocked(fs.writeFileSync)).toHaveBeenCalled()
|
||||
expect(vi.mocked(fs.readFileSync)).toHaveBeenCalled()
|
||||
expect(vi.mocked(fs.unlinkSync)).toHaveBeenCalled()
|
||||
expect(res.embedding).toEqual([0.1, 0.2, 0.3])
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith(
|
||||
'shumai',
|
||||
'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate text embedding successfully', async () => {
|
||||
@@ -291,6 +276,63 @@ describe('AI Database Activities Integration', () => {
|
||||
expect(ctx.asset.id).toBe(asset.id)
|
||||
expect(ctx.asset.mediaType).toBe('video/mp4')
|
||||
})
|
||||
|
||||
it('should parse valid positive chunk duration and fallback on invalid or negative ones', async () => {
|
||||
// Create embedding agent if not already present
|
||||
const existingAgent = await prisma.agent.findUnique({ where: { id: user.id } })
|
||||
if (!existingAgent) {
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: user.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: {
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Test default/fallback
|
||||
delete process.env.EMBEDDING_CHUNK_DURATION
|
||||
let ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(60.0)
|
||||
expect(ctx.chunkOverlap).toBe(5.0)
|
||||
|
||||
// Test valid positive
|
||||
process.env.EMBEDDING_CHUNK_DURATION = '45.5'
|
||||
process.env.EMBEDDING_CHUNK_OVERLAP = '10.0'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(45.5)
|
||||
expect(ctx.chunkOverlap).toBe(10.0)
|
||||
|
||||
// Test negative/invalid for overlap
|
||||
process.env.EMBEDDING_CHUNK_OVERLAP = '-5.0'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkOverlap).toBe(5.0)
|
||||
|
||||
process.env.EMBEDDING_CHUNK_OVERLAP = 'invalid'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkOverlap).toBe(5.0)
|
||||
|
||||
// Cleanup env
|
||||
delete process.env.EMBEDDING_CHUNK_DURATION
|
||||
delete process.env.EMBEDDING_CHUNK_OVERLAP
|
||||
})
|
||||
})
|
||||
|
||||
describe('saveAssetEmbeddingsActivity', () => {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GoogleGenAI } from '@google/genai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/transcode'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { execFile } from 'child_process'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
@@ -140,6 +140,8 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
|
||||
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`)
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'warning',
|
||||
'-i',
|
||||
tmpFile,
|
||||
'-ss',
|
||||
@@ -309,9 +311,18 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
|
||||
throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true })
|
||||
}
|
||||
|
||||
const parsedDuration = parseFloat(process.env.EMBEDDING_CHUNK_DURATION || '')
|
||||
const chunkDuration =
|
||||
Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0
|
||||
|
||||
const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '')
|
||||
const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
|
||||
|
||||
return {
|
||||
agent,
|
||||
asset,
|
||||
chunkDuration,
|
||||
chunkOverlap,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -338,3 +349,91 @@ export async function saveAssetEmbeddingsActivity(params: {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface GenerateImageEmbeddingParams {
|
||||
teamId: string
|
||||
assetKey: string
|
||||
mediaType: string
|
||||
}
|
||||
|
||||
export async function generateImageEmbeddingActivity(
|
||||
params: GenerateImageEmbeddingParams,
|
||||
): Promise<{
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'GEMINI_API_KEY environment variable is not configured',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const resolvedApiKey = process.env[apiKey] || apiKey
|
||||
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
|
||||
|
||||
const usage: Usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.assetKey,
|
||||
)
|
||||
const embVec = await generateMultimodalEmbedding(ai, data, params.mediaType)
|
||||
|
||||
return {
|
||||
embedding: embVec,
|
||||
usage,
|
||||
}
|
||||
}
|
||||
|
||||
export interface GenerateVideoChunkEmbeddingParams {
|
||||
teamId: string
|
||||
chunkKey: string
|
||||
}
|
||||
|
||||
export async function generateVideoChunkEmbeddingActivity(
|
||||
params: GenerateVideoChunkEmbeddingParams,
|
||||
): Promise<{
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'GEMINI_API_KEY environment variable is not configured',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const resolvedApiKey = process.env[apiKey] || apiKey
|
||||
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
|
||||
|
||||
const usage: Usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
try {
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.chunkKey,
|
||||
)
|
||||
|
||||
const embVec = await generateMultimodalEmbedding(ai, data, 'video/mp4')
|
||||
return {
|
||||
embedding: embVec,
|
||||
usage,
|
||||
}
|
||||
} catch (err) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `Failed to generate video chunk embedding for key ${params.chunkKey}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
nonRetryable: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -316,6 +316,7 @@ describe('DatabaseSessionStorage', () => {
|
||||
|
||||
// 6. Resume the session via createAgentSession
|
||||
await createAgentSession({
|
||||
teamId: 't1',
|
||||
agentId: agent.id,
|
||||
providerName: 'test-provider',
|
||||
modelId: 'test-model',
|
||||
@@ -334,6 +335,7 @@ describe('DatabaseSessionStorage', () => {
|
||||
expect.objectContaining({
|
||||
MOCK_ENV_VAR: 'mocked-value',
|
||||
}),
|
||||
expect.any(Object),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -424,6 +426,7 @@ describe('DatabaseSessionStorage', () => {
|
||||
|
||||
// 3. Resume the session via createAgentSession with a NEW comment ID
|
||||
await createAgentSession({
|
||||
teamId: 't1',
|
||||
agentId: agent.id,
|
||||
providerName: 'test-provider',
|
||||
modelId: 'test-model',
|
||||
|
||||
@@ -1,6 +1,27 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { formatSkillsForPrompt } from './index'
|
||||
import { describe, it, expect, vi } from 'vitest'
|
||||
import { formatSkillsForPrompt, createAgentSession } from './index'
|
||||
import { createSandboxedBashTool } from './tools/sandboxed-bash'
|
||||
import { SandboxManager, type SandboxAskCallback } from '@anthropic-ai/sandbox-runtime'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import * as path from 'path'
|
||||
import * as childProcess from 'node:child_process'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let mockSpawn: any = null
|
||||
|
||||
vi.mock('node:child_process', async () => {
|
||||
const actual = (await vi.importActual('node:child_process')) as typeof childProcess
|
||||
return {
|
||||
...actual,
|
||||
spawn: (...args: Parameters<typeof childProcess.spawn>) => {
|
||||
if (mockSpawn) {
|
||||
return mockSpawn(...args)
|
||||
}
|
||||
return actual.spawn(...args)
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
describe('formatSkillsForPrompt', () => {
|
||||
it('should return empty string if skills array is empty', () => {
|
||||
@@ -31,3 +52,281 @@ describe('formatSkillsForPrompt', () => {
|
||||
expect(result).toContain('</available_skills>')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Sandbox Network isolation integration', () => {
|
||||
setupTestDbHooks()
|
||||
|
||||
it('should register callback that pushes blocked hosts to database and propagates to bash tool', async () => {
|
||||
// 1. Create a team
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Sandbox Team' },
|
||||
})
|
||||
|
||||
// Ensure agent user and agent exist in database to avoid foreign key violation
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'test-agent',
|
||||
name: 'Ai Agent',
|
||||
email: 'test-agent@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'test-agent',
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: {
|
||||
provider: 'openai',
|
||||
model: 'gpt-4',
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Mock / spy SandboxManager
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
const wrapSpy = vi
|
||||
.spyOn(SandboxManager, 'wrapWithSandbox')
|
||||
.mockImplementation(async (cmd) => cmd)
|
||||
|
||||
// 3. Call createAgentSession
|
||||
await createAgentSession({
|
||||
teamId: team.id,
|
||||
agentId: 'test-agent',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: [],
|
||||
})
|
||||
|
||||
// 4. Verify SandboxManager.initialize was called and extract the callback
|
||||
expect(initializeSpy).toHaveBeenCalled()
|
||||
const callback = initializeSpy.mock.calls[0][1]
|
||||
expect(callback).toBeDefined()
|
||||
|
||||
// 5. Trigger the callback with a blocked host
|
||||
const result = await callback!({ host: 'blocked-api.com', port: 443 })
|
||||
expect(result).toBe(false) // should always return false to block
|
||||
|
||||
// 6. Verify it was pushed to database
|
||||
const sandbox = await prisma.sandbox.findUnique({
|
||||
where: { teamId: team.id },
|
||||
})
|
||||
expect(sandbox?.pendingDomains).toContain('blocked-api.com')
|
||||
|
||||
// 7. Verify that if the bash tool is run while a domain is blocked, it rejects with correct error
|
||||
const sandboxState = { blockedHost: '' }
|
||||
const bashTool = createSandboxedBashTool(
|
||||
process.cwd(),
|
||||
{},
|
||||
{
|
||||
getBlockedHost: () => sandboxState.blockedHost,
|
||||
clearBlockedHost: () => {
|
||||
sandboxState.blockedHost = ''
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// Mock spawn to simulate process exit and trigger network block
|
||||
mockSpawn = () => {
|
||||
sandboxState.blockedHost = 'blocked-api.com'
|
||||
const mockChild = {
|
||||
pid: 123,
|
||||
stdout: { on: vi.fn() },
|
||||
stderr: { on: vi.fn() },
|
||||
on: vi.fn((event: string, cb: (code: number) => void) => {
|
||||
if (event === 'close') {
|
||||
setTimeout(() => cb(1), 10)
|
||||
}
|
||||
}),
|
||||
} as unknown as childProcess.ChildProcess
|
||||
return mockChild
|
||||
}
|
||||
|
||||
const bashPromise = bashTool.execute('1', { command: 'curl blocked-api.com' })
|
||||
|
||||
try {
|
||||
await expect(bashPromise).rejects.toThrow(
|
||||
'Network request to blocked-api.com is blocked, please ask admin to allow it in sandbox settings.',
|
||||
)
|
||||
} finally {
|
||||
mockSpawn = null
|
||||
initializeSpy.mockRestore()
|
||||
wrapSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('should correctly route sandbox callback to the active session when multiple sessions are created sequentially', async () => {
|
||||
// 1. Create two teams
|
||||
const team1 = await prisma.team.create({
|
||||
data: { name: 'Team 1' },
|
||||
})
|
||||
const team2 = await prisma.team.create({
|
||||
data: { name: 'Team 2' },
|
||||
})
|
||||
|
||||
// Ensure agent user and agents exist in database
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'agent-1',
|
||||
name: 'Ai Agent 1',
|
||||
email: 'agent1@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'agent-2',
|
||||
name: 'Ai Agent 2',
|
||||
email: 'agent2@shumai.ai',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'agent-1',
|
||||
teamId: team1.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'openai', model: 'gpt-4' },
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'agent-2',
|
||||
teamId: team2.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'openai', model: 'gpt-4' },
|
||||
},
|
||||
})
|
||||
|
||||
// Mock SandboxManager.initialize to behave like a process-global singleton
|
||||
let registeredCallback: SandboxAskCallback | null = null
|
||||
const initializeSpy = vi
|
||||
.spyOn(SandboxManager, 'initialize')
|
||||
.mockImplementation(async (config, cb) => {
|
||||
if (!registeredCallback) {
|
||||
registeredCallback = cb || null
|
||||
}
|
||||
})
|
||||
const resetSpy = vi.spyOn(SandboxManager, 'reset').mockImplementation(async () => {
|
||||
// In the real SandboxManager, reset clears the global initialization promise and servers,
|
||||
// which allows subsequent initialize calls to register a new callback.
|
||||
registeredCallback = null
|
||||
})
|
||||
|
||||
try {
|
||||
// 2. Call createAgentSession for Session 1
|
||||
await createAgentSession({
|
||||
teamId: team1.id,
|
||||
agentId: 'agent-1',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: [],
|
||||
})
|
||||
|
||||
// 3. Call createAgentSession for Session 2
|
||||
await createAgentSession({
|
||||
teamId: team2.id,
|
||||
agentId: 'agent-2',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: [],
|
||||
})
|
||||
|
||||
expect(registeredCallback).toBeDefined()
|
||||
|
||||
// 4. Trigger the globally registered callback (which is from Session 1)
|
||||
const result = await registeredCallback!({ host: 'blocked-for-session-2.com', port: 443 })
|
||||
expect(result).toBe(false)
|
||||
|
||||
// 5. Verify that the pending domain was pushed to the DB record of Team 2
|
||||
const sandbox2 = await prisma.sandbox.findUnique({
|
||||
where: { teamId: team2.id },
|
||||
})
|
||||
expect(sandbox2?.pendingDomains).toContain('blocked-for-session-2.com')
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
resetSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('should pass thinkingLevel to AgentHarness and default to off', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team' },
|
||||
})
|
||||
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'test-agent-1',
|
||||
name: 'Agent 1',
|
||||
email: 'agent1@test.com',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'test-agent-1',
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'google', model: 'gemini' },
|
||||
},
|
||||
})
|
||||
|
||||
await prisma.user.create({
|
||||
data: {
|
||||
id: 'test-agent-2',
|
||||
name: 'Agent 2',
|
||||
email: 'agent2@test.com',
|
||||
type: 'agent',
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: 'test-agent-2',
|
||||
teamId: team.id,
|
||||
type: 'chat',
|
||||
config: { provider: 'google', model: 'gemini' },
|
||||
},
|
||||
})
|
||||
|
||||
const initializeSpy = vi.spyOn(SandboxManager, 'initialize').mockResolvedValue()
|
||||
try {
|
||||
// Call createAgentSession with thinkingLevel
|
||||
const { harness: harnessHigh } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
agentId: 'test-agent-1',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: [],
|
||||
thinkingLevel: 'high',
|
||||
})
|
||||
expect(harnessHigh.getThinkingLevel()).toBe('high')
|
||||
|
||||
// Call createAgentSession without thinkingLevel
|
||||
const { harness: harnessDefault } = await createAgentSession({
|
||||
teamId: team.id,
|
||||
agentId: 'test-agent-2',
|
||||
providerName: 'google',
|
||||
modelId: 'gemini',
|
||||
systemPrompt: 'prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
providers: [],
|
||||
})
|
||||
expect(harnessDefault.getThinkingLevel()).toBe('off')
|
||||
} finally {
|
||||
initializeSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
import { SandboxManager } from '@anthropic-ai/sandbox-runtime'
|
||||
import { AgentHarness, Session, type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import {
|
||||
AgentHarness,
|
||||
Session,
|
||||
type AgentTool,
|
||||
type ThinkingLevel,
|
||||
} from '@earendil-works/pi-agent-core'
|
||||
import { NodeExecutionEnv } from '@earendil-works/pi-agent-core/node'
|
||||
import { getModel } from '@earendil-works/pi-ai'
|
||||
import { Type, type TSchema } from '@sinclair/typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { createAnalyzeImageTool } from './tools/analyze-image'
|
||||
@@ -17,6 +23,7 @@ import { createReadSkillTool } from './tools/read-skill'
|
||||
import { createSandboxedBashTool } from './tools/sandboxed-bash'
|
||||
|
||||
export interface CreateAgentSessionParams {
|
||||
teamId: string
|
||||
agentId: string
|
||||
providerName: string
|
||||
modelId: string
|
||||
@@ -27,6 +34,7 @@ export interface CreateAgentSessionParams {
|
||||
userId?: string
|
||||
userCommentId?: string | null
|
||||
customTools?: AgentTool[]
|
||||
thinkingLevel?: string
|
||||
providers: Array<{
|
||||
name: string
|
||||
config: PrismaJson.ProviderConfig
|
||||
@@ -40,6 +48,7 @@ export interface CreateAgentSessionParams {
|
||||
|
||||
export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
const {
|
||||
teamId,
|
||||
agentId,
|
||||
providerName,
|
||||
modelId,
|
||||
@@ -50,6 +59,7 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
userId,
|
||||
userCommentId: passedUserCommentId,
|
||||
customTools = [],
|
||||
thinkingLevel,
|
||||
} = params
|
||||
|
||||
const storage = await DatabaseSessionStorage.create({
|
||||
@@ -115,18 +125,57 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
|
||||
const allowWrite = [piDir]
|
||||
|
||||
const sandboxState = {
|
||||
blockedHost: '',
|
||||
}
|
||||
|
||||
const sandboxAskCallback = async ({ host }: { host: string; port?: number }) => {
|
||||
try {
|
||||
const sandbox = await prisma.sandbox.findUnique({
|
||||
where: { teamId },
|
||||
})
|
||||
const pendingDomains = sandbox?.pendingDomains || []
|
||||
if (!pendingDomains.includes(host)) {
|
||||
await prisma.sandbox.upsert({
|
||||
where: { teamId },
|
||||
create: {
|
||||
teamId,
|
||||
pendingDomains: [host],
|
||||
},
|
||||
update: {
|
||||
pendingDomains: {
|
||||
push: host,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to update sandbox pending domains:', err)
|
||||
}
|
||||
|
||||
sandboxState.blockedHost = host
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// SandboxManager.initialize is a global operation that applies to the entire process.
|
||||
await SandboxManager.initialize({
|
||||
network: {
|
||||
allowedDomains,
|
||||
deniedDomains: [],
|
||||
// We reset it first to ensure any previous process-global SOCKS/HTTP proxy servers
|
||||
// and callbacks are cleaned up, allowing the new callback and allowedDomains to take effect.
|
||||
await SandboxManager.reset()
|
||||
await SandboxManager.initialize(
|
||||
{
|
||||
network: {
|
||||
allowedDomains,
|
||||
deniedDomains: [],
|
||||
},
|
||||
filesystem: {
|
||||
denyRead: ['.env', '.env.*', '*.pem', '*.key'],
|
||||
allowWrite,
|
||||
denyWrite: ['.env', '.env.*', '*.pem', '*.key'],
|
||||
},
|
||||
},
|
||||
filesystem: {
|
||||
denyRead: ['.env', '.env.*', '*.pem', '*.key'],
|
||||
allowWrite,
|
||||
denyWrite: ['.env', '.env.*', '*.pem', '*.key'],
|
||||
},
|
||||
})
|
||||
sandboxAskCallback,
|
||||
)
|
||||
|
||||
const skillEnvs: Record<string, string> = {}
|
||||
const onEnvsAdded = (envs: Record<string, string>) => {
|
||||
@@ -169,7 +218,12 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
)
|
||||
}
|
||||
|
||||
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs)
|
||||
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs, {
|
||||
getBlockedHost: () => sandboxState.blockedHost,
|
||||
clearBlockedHost: () => {
|
||||
sandboxState.blockedHost = ''
|
||||
},
|
||||
})
|
||||
const readSkill = createReadSkillTool(onEnvsAdded)
|
||||
|
||||
const systemTools: AgentTool[] = []
|
||||
@@ -186,6 +240,7 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
env: new NodeExecutionEnv({ cwd: process.cwd() }),
|
||||
session,
|
||||
model,
|
||||
thinkingLevel: (thinkingLevel || 'off') as ThinkingLevel,
|
||||
systemPrompt: async () => {
|
||||
let prompt = systemPrompt
|
||||
|
||||
@@ -333,6 +388,8 @@ export {
|
||||
generateTextEmbeddingActivity,
|
||||
type ExtractAiMetadataParams,
|
||||
extractAiMetadataActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
} from './activities/ai'
|
||||
export * from './workflows/agent-autofill'
|
||||
export * from './workflows/agent-chat'
|
||||
|
||||
@@ -15,9 +15,15 @@ type BashDetails = {
|
||||
stderr: string
|
||||
}
|
||||
|
||||
export interface SandboxedBashOptions {
|
||||
getBlockedHost?: () => string
|
||||
clearBlockedHost?: () => void
|
||||
}
|
||||
|
||||
export const createSandboxedBashTool = (
|
||||
cwd: string,
|
||||
skillEnvs: Record<string, string> = {},
|
||||
options?: SandboxedBashOptions,
|
||||
): AgentTool<typeof BashParameters, BashDetails> => {
|
||||
return {
|
||||
name: 'bash',
|
||||
@@ -29,6 +35,7 @@ export const createSandboxedBashTool = (
|
||||
{ command, timeout },
|
||||
signal,
|
||||
): Promise<AgentToolResult<BashDetails>> => {
|
||||
options?.clearBlockedHost?.()
|
||||
const wrappedCommand = await SandboxManager.wrapWithSandbox(command)
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -94,12 +101,20 @@ export const createSandboxedBashTool = (
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle)
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
|
||||
const blocked = options?.getBlockedHost?.()
|
||||
|
||||
if (signal?.aborted) {
|
||||
logger.error({ command, stdout, stderr }, 'Sandboxed bash command aborted')
|
||||
reject(new Error('aborted'))
|
||||
} else if (timedOut) {
|
||||
logger.error({ timeout, command, stdout, stderr }, 'Sandboxed bash command timed out')
|
||||
reject(new Error(`bash command timed out after ${timeout} seconds`))
|
||||
} else if (blocked) {
|
||||
reject(
|
||||
new Error(
|
||||
`Network request to ${blocked} is blocked, please ask admin to allow it in sandbox settings.`,
|
||||
),
|
||||
)
|
||||
} else if (code !== 0 && code !== null) {
|
||||
logger.error(
|
||||
{ exitCode: code, command, stdout, stderr },
|
||||
|
||||
@@ -29,8 +29,11 @@ describe('Agent Embedding Workflow', () => {
|
||||
getEmbeddingContextActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getEmbeddingContextActivity',
|
||||
}),
|
||||
generateEmbeddingActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'generateEmbeddingActivity',
|
||||
generateImageEmbeddingActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'generateImageEmbeddingActivity',
|
||||
}),
|
||||
generateVideoChunkEmbeddingActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'generateVideoChunkEmbeddingActivity',
|
||||
}),
|
||||
saveAssetEmbeddingsActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'saveAssetEmbeddingsActivity',
|
||||
@@ -41,20 +44,58 @@ describe('Agent Embedding Workflow', () => {
|
||||
getAgentWorkerQueueActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getAgentWorkerQueueActivity',
|
||||
}),
|
||||
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'getTranscodeWorkerQueueActivity',
|
||||
}),
|
||||
downloadMediaToTmpActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'downloadMediaToTmpActivity',
|
||||
}),
|
||||
cleanupTmpDirActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'cleanupTmpDirActivity',
|
||||
}),
|
||||
transcodeVideoChunkActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'transcodeVideoChunkActivity',
|
||||
}),
|
||||
deleteS3ObjectActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'deleteS3ObjectActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
mockActivities.getAgentWorkerQueueActivity.mockResolvedValue('agent_queue')
|
||||
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_queue')
|
||||
mockActivities.updateTaskStatusActivity.mockResolvedValue({})
|
||||
mockActivities.createCommentActivity.mockResolvedValue({ id: 'comment-placeholder-id' })
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: { id: 'a1' },
|
||||
dbProvider: { name: 'google' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'test.png' },
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
mockActivities.generateEmbeddingActivity.mockResolvedValue({
|
||||
embeddings: [{ embedding: [0.1, 0.2] }],
|
||||
mockActivities.generateImageEmbeddingActivity.mockResolvedValue({
|
||||
embedding: [0.1, 0.2],
|
||||
usage: { inputTokens: 5, outputTokens: 5, model: 'gpt' },
|
||||
})
|
||||
mockActivities.generateVideoChunkEmbeddingActivity.mockResolvedValue({
|
||||
embedding: [0.3, 0.4],
|
||||
usage: { inputTokens: 3, outputTokens: 3, model: 'gpt' },
|
||||
})
|
||||
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
|
||||
filePath: '/tmp/test.mp4',
|
||||
tmpDir: '/tmp/test-dir',
|
||||
})
|
||||
mockActivities.transcodeVideoChunkActivity.mockImplementation(
|
||||
async (params: { assetId: string; startTime: number; endTime: number }) => {
|
||||
return {
|
||||
chunkKey: `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}.mp4`,
|
||||
}
|
||||
},
|
||||
)
|
||||
mockActivities.deleteS3ObjectActivity.mockResolvedValue(undefined)
|
||||
mockActivities.cleanupTmpDirActivity.mockResolvedValue(undefined)
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities contains vi.fn mock functions which are cast to expected activity proxy types
|
||||
vi.mocked(workflowUtils.getActivities).mockReturnValue(mockActivities as any)
|
||||
@@ -64,7 +105,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow successfully', async () => {
|
||||
it('should run agent embedding workflow successfully for image', async () => {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
@@ -103,15 +144,11 @@ describe('Agent Embedding Workflow', () => {
|
||||
assetId: 'a1',
|
||||
})
|
||||
|
||||
// Verify embedding generation
|
||||
expect(mockActivities.generateEmbeddingActivity).toHaveBeenCalledWith({
|
||||
// Verify image embedding generation
|
||||
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({
|
||||
teamId: 't1',
|
||||
assetId: 'a1',
|
||||
context: {
|
||||
agent: { id: 'b1' },
|
||||
asset: { id: 'a1' },
|
||||
dbProvider: { name: 'google' },
|
||||
},
|
||||
assetKey: 'test.png',
|
||||
mediaType: 'image/png',
|
||||
})
|
||||
|
||||
// Verify embeddings saved
|
||||
@@ -125,7 +162,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
taskId: task.id,
|
||||
inputTokens: 5,
|
||||
outputTokens: 5,
|
||||
model: 'gpt',
|
||||
model: 'gemini-embedding-2',
|
||||
})
|
||||
|
||||
// Verify placeholder comment updated
|
||||
@@ -141,6 +178,142 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow successfully for image with transcoded format', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'test.png' },
|
||||
media: {
|
||||
imageTranscodes: [{ key: 'test-transcoded.webp', isRaw: false, format: 'webp' }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({
|
||||
teamId: 't1',
|
||||
assetKey: 'test-transcoded.webp',
|
||||
mediaType: 'image/webp',
|
||||
})
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow successfully for video in chunks', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 150.0,
|
||||
videoTranscodes: [{ key: 'test-transcoded.mp4', isRaw: false }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
payload: {
|
||||
projectId: 'p1',
|
||||
agent: { sessionId: 's1', agentId: 'agent-1' },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Verify video transcode queue discovery & download
|
||||
expect(mockActivities.getTranscodeWorkerQueueActivity).toHaveBeenCalled()
|
||||
expect(mockActivities.downloadMediaToTmpActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
})
|
||||
|
||||
// Verify video chunk slicing activity called 3 times on transcode queue
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(3)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(1, {
|
||||
assetId: 'a1',
|
||||
filePath: '/tmp/test.mp4',
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
|
||||
assetId: 'a1',
|
||||
filePath: '/tmp/test.mp4',
|
||||
startTime: 55,
|
||||
endTime: 115,
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
|
||||
assetId: 'a1',
|
||||
filePath: '/tmp/test.mp4',
|
||||
startTime: 110,
|
||||
endTime: 150,
|
||||
})
|
||||
|
||||
// Verify video chunk embedding generation (3 chunks) on agent queue
|
||||
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenCalledTimes(3)
|
||||
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(1, {
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
|
||||
})
|
||||
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, {
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-55-115.mp4',
|
||||
})
|
||||
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, {
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-110-150.mp4',
|
||||
})
|
||||
|
||||
// Verify deletion of temporary chunks from S3
|
||||
expect(mockActivities.deleteS3ObjectActivity).toHaveBeenCalledTimes(3)
|
||||
expect(mockActivities.deleteS3ObjectActivity).toHaveBeenNthCalledWith(1, {
|
||||
key: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
|
||||
})
|
||||
|
||||
// Verify temp directory cleanup on transcode queue
|
||||
expect(mockActivities.cleanupTmpDirActivity).toHaveBeenCalledWith({
|
||||
tmpDir: '/tmp/test-dir',
|
||||
})
|
||||
|
||||
// Verify usage update (3 chunks * 3 tokens = 9)
|
||||
expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
inputTokens: 9,
|
||||
outputTokens: 9,
|
||||
model: 'gemini-embedding-2',
|
||||
})
|
||||
|
||||
// Verify embeddings saved
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
embeddings: [
|
||||
{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 },
|
||||
{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 },
|
||||
{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw if task has no teamId', async () => {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
@@ -160,10 +333,19 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should skip saving embeddings if none are returned', async () => {
|
||||
mockActivities.generateEmbeddingActivity.mockResolvedValue({
|
||||
embeddings: [],
|
||||
usage: null,
|
||||
it('should skip saving embeddings if none are returned (video duration 0)', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 0.0,
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
@@ -178,7 +360,6 @@ describe('Agent Embedding Workflow', () => {
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateTaskUsageActivity).not.toHaveBeenCalled()
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: task.id,
|
||||
status: 'completed',
|
||||
@@ -186,7 +367,9 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
|
||||
it('should handle failures, update placeholder with error, and set status to failed', async () => {
|
||||
mockActivities.generateEmbeddingActivity.mockRejectedValue(new Error('AI Service Unavailable'))
|
||||
mockActivities.generateImageEmbeddingActivity.mockRejectedValue(
|
||||
new Error('AI Service Unavailable'),
|
||||
)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
@@ -212,4 +395,40 @@ describe('Agent Embedding Workflow', () => {
|
||||
output: { error: 'AI Service Unavailable' },
|
||||
})
|
||||
})
|
||||
|
||||
it('should not create redundant chunks when video exactly matches chunk duration', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 60.0,
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Verify only 1 chunk was created instead of 2 (0-60 and 55-60)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(1)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,25 +1,48 @@
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core'
|
||||
import {
|
||||
getActivities,
|
||||
executeActivity,
|
||||
TaskQueueAgent,
|
||||
TaskQueueTranscode,
|
||||
} from '@shumai/workflow-core'
|
||||
|
||||
export interface GeneratedEmbedding {
|
||||
embedding: number[]
|
||||
startTime?: number
|
||||
endTime?: number
|
||||
}
|
||||
|
||||
export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
const {
|
||||
updateTaskStatusActivity,
|
||||
getEmbeddingContextActivity,
|
||||
generateEmbeddingActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
updateTaskUsageActivity,
|
||||
createCommentActivity,
|
||||
updateCommentActivity,
|
||||
getAgentWorkerQueueActivity,
|
||||
getTranscodeWorkerQueueActivity,
|
||||
downloadMediaToTmpActivity,
|
||||
cleanupTmpDirActivity,
|
||||
transcodeVideoChunkActivity,
|
||||
deleteS3ObjectActivity,
|
||||
} = getActivities()
|
||||
|
||||
let placeholderCommentId: string | undefined
|
||||
let agentWorkerQueue = ''
|
||||
let transcodeWorkerQueue = ''
|
||||
let tmpDir: string | undefined
|
||||
|
||||
try {
|
||||
// 0. Discover queue
|
||||
// 0. Discover queues
|
||||
agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity)
|
||||
transcodeWorkerQueue = await executeActivity(
|
||||
TaskQueueTranscode,
|
||||
getTranscodeWorkerQueueActivity,
|
||||
)
|
||||
|
||||
// Update status to processing
|
||||
await executeActivity(agentWorkerQueue, updateTaskStatusActivity, {
|
||||
@@ -47,30 +70,153 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
assetId: task.assetId,
|
||||
})
|
||||
|
||||
// Call the activity to generate embeddings
|
||||
const result = await executeActivity(agentWorkerQueue, generateEmbeddingActivity, {
|
||||
teamId: task.teamId,
|
||||
assetId: task.assetId,
|
||||
context,
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const { asset, chunkDuration, chunkOverlap } = context as any
|
||||
|
||||
const isImage = asset.mediaType?.startsWith('image/')
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `unsupported media type for embeddings: ${asset.mediaType}`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Resolve the S3 key to use for embedding (preferring transcoded version)
|
||||
let embeddingKey = asset.storageKey?.key
|
||||
let embeddingMediaType = asset.mediaType
|
||||
if (asset.media) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const media = asset.media as any
|
||||
if (isVideo) {
|
||||
if (media.videoTranscodes && media.videoTranscodes.length > 0) {
|
||||
// Find first non-raw transcode or fallback to first transcode
|
||||
const transcode =
|
||||
media.videoTranscodes.find((t: PrismaJson.VideoTranscode) => !t.isRaw) ||
|
||||
media.videoTranscodes[0]
|
||||
if (transcode?.key) {
|
||||
embeddingKey = transcode.key
|
||||
}
|
||||
}
|
||||
} else if (isImage) {
|
||||
if (media.imageTranscodes && media.imageTranscodes.length > 0) {
|
||||
// Find first non-raw transcode or fallback to first transcode
|
||||
const transcode =
|
||||
media.imageTranscodes.find((t: PrismaJson.ImageTranscode) => !t.isRaw) ||
|
||||
media.imageTranscodes[0]
|
||||
if (transcode?.key) {
|
||||
embeddingKey = transcode.key
|
||||
if (transcode.format) {
|
||||
embeddingMediaType = `image/${transcode.format}`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!embeddingKey) {
|
||||
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
|
||||
}
|
||||
|
||||
const results: GeneratedEmbedding[] = []
|
||||
const usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
if (isImage) {
|
||||
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
|
||||
teamId: task.teamId,
|
||||
assetKey: embeddingKey,
|
||||
mediaType: embeddingMediaType,
|
||||
})
|
||||
results.push({ embedding: result.embedding })
|
||||
if (result.usage) {
|
||||
usage.inputTokens += result.usage.inputTokens || 0
|
||||
usage.outputTokens += result.usage.outputTokens || 0
|
||||
}
|
||||
} else if (isVideo) {
|
||||
// 1. Download full video to transcode worker temp space
|
||||
const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, {
|
||||
assetKey: embeddingKey,
|
||||
})
|
||||
const { filePath } = download
|
||||
tmpDir = download.tmpDir
|
||||
|
||||
// Get video duration
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const duration = (asset.media as any)?.duration || 0
|
||||
const limit = chunkDuration || 60.0
|
||||
const overlap = chunkOverlap ?? 5.0
|
||||
const step = Math.max(1, limit - overlap)
|
||||
|
||||
for (let start = 0.0; start < duration; start += step) {
|
||||
let end = start + limit
|
||||
if (end >= duration) {
|
||||
end = duration
|
||||
}
|
||||
|
||||
// Step A: Slice chunk on transcode worker and upload to S3
|
||||
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
|
||||
assetId: asset.id,
|
||||
filePath,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
})
|
||||
const { chunkKey } = chunkRes
|
||||
|
||||
try {
|
||||
// Step B: Generate embedding on agent worker
|
||||
const chunkResult = await executeActivity(
|
||||
agentWorkerQueue,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
{
|
||||
teamId: task.teamId,
|
||||
chunkKey,
|
||||
},
|
||||
)
|
||||
|
||||
results.push({
|
||||
embedding: chunkResult.embedding,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
})
|
||||
|
||||
if (chunkResult.usage) {
|
||||
usage.inputTokens += chunkResult.usage.inputTokens || 0
|
||||
usage.outputTokens += chunkResult.usage.outputTokens || 0
|
||||
}
|
||||
} finally {
|
||||
// Step C: Delete temporary chunk from S3
|
||||
await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, {
|
||||
key: chunkKey,
|
||||
})
|
||||
}
|
||||
|
||||
// If this chunk reached the end of the video, stop to avoid redundant overlapping chunks
|
||||
if (end >= duration) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save computed embeddings
|
||||
if (result.embeddings.length > 0) {
|
||||
if (results.length > 0) {
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: result.embeddings,
|
||||
embeddings: results,
|
||||
})
|
||||
}
|
||||
|
||||
// Update Usage
|
||||
if (result.usage) {
|
||||
await executeActivity(agentWorkerQueue, updateTaskUsageActivity, {
|
||||
taskId: task.id,
|
||||
inputTokens: result.usage.inputTokens,
|
||||
outputTokens: result.usage.outputTokens,
|
||||
model: result.usage.model,
|
||||
})
|
||||
}
|
||||
await executeActivity(agentWorkerQueue, updateTaskUsageActivity, {
|
||||
taskId: task.id,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
model: usage.model,
|
||||
})
|
||||
|
||||
// 7. Update Placeholder Comment
|
||||
if (placeholderCommentId) {
|
||||
@@ -109,6 +255,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
})
|
||||
}
|
||||
throw err
|
||||
} finally {
|
||||
if (tmpDir && transcodeWorkerQueue) {
|
||||
try {
|
||||
await executeActivity(transcodeWorkerQueue, cleanupTmpDirActivity, { tmpDir })
|
||||
} catch (cleanupErr) {
|
||||
console.error('Failed to cleanup transcode worker tmp dir:', cleanupErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.0.2",
|
||||
"version": "0.1.3",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -39,7 +39,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
avatar: (await getAvatarUrl(agent.user.image)) || undefined,
|
||||
providerId: agent.providerId || undefined,
|
||||
modelId: agent.modelId || undefined,
|
||||
thinkingLevel: config.thinkingLevel || '',
|
||||
thinkingLevel: config.thinkingLevel || 'off',
|
||||
systemPrompt: config.systemPrompt,
|
||||
soul: agent.soul || undefined,
|
||||
skills: agent.skills.map((s) => ({
|
||||
@@ -81,7 +81,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
avatar: (await getAvatarUrl(agent.user.image)) || undefined,
|
||||
providerId: agent.providerId || undefined,
|
||||
modelId: agent.modelId || undefined,
|
||||
thinkingLevel: config.thinkingLevel || '',
|
||||
thinkingLevel: config.thinkingLevel || 'off',
|
||||
systemPrompt: config.systemPrompt,
|
||||
soul: agent.soul || undefined,
|
||||
skills: agent.skills.map((s) => ({
|
||||
@@ -119,7 +119,7 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
avatar: (await getAvatarUrl(agent.user.image)) || undefined,
|
||||
providerId: agent.providerId || undefined,
|
||||
modelId: agent.modelId || undefined,
|
||||
thinkingLevel: config.thinkingLevel || '',
|
||||
thinkingLevel: config.thinkingLevel || 'off',
|
||||
systemPrompt: config.systemPrompt,
|
||||
soul: agent.soul || undefined,
|
||||
skills: agent.skills.map((s) => ({
|
||||
@@ -17,7 +17,8 @@ describe('authn api', () => {
|
||||
|
||||
it('GET /signup-info', async () => {
|
||||
mockGetSignupInfo.mockResolvedValue({
|
||||
userCount: 5,
|
||||
initialized: true,
|
||||
demoMode: false,
|
||||
})
|
||||
|
||||
const app = new Hono().route('/', authnRoute)
|
||||
@@ -25,7 +26,8 @@ describe('authn api', () => {
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.json()).toEqual({
|
||||
userCount: 5,
|
||||
initialized: true,
|
||||
demoMode: false,
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -17,7 +17,8 @@ const route = app
|
||||
.get('/signup-info', async (c) => {
|
||||
const info = await teamService.getSignupInfo()
|
||||
return c.json({
|
||||
userCount: info.userCount,
|
||||
initialized: info.initialized,
|
||||
demoMode: info.demoMode,
|
||||
})
|
||||
})
|
||||
.get('/me', async (c) => {
|
||||
@@ -6,7 +6,7 @@ import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/transcode/src/transcode'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { authMiddleware } from './middleware/auth'
|
||||
import { authzService, ResourceType, Permission } from '@shumai/core/src/authz/authz'
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
uploadFileRequestSchema,
|
||||
getDownloadLinksRequestSchema,
|
||||
} from '@shumai/dtos'
|
||||
import { transcodeService } from '@shumai/transcode/src/transcode'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import fs from 'fs'
|
||||
import { Hono } from 'hono'
|
||||
import os from 'os'
|
||||
@@ -1,25 +1,26 @@
|
||||
import type { Prisma } from '@shumai/db'
|
||||
import { Hono } from 'hono'
|
||||
import agentRoute from './api/agent'
|
||||
import attachmentRoute from './api/attachment'
|
||||
import authnRoute from './api/authn'
|
||||
import collectionRoute from './api/collection'
|
||||
import fileRoute from './api/file'
|
||||
import folderRoute from './api/folder'
|
||||
import inviteRoute from './api/invite'
|
||||
import metadataRoute from './api/metadata'
|
||||
import { authMiddleware } from './api/middleware/auth'
|
||||
import notificationRoute from './api/notification'
|
||||
import projectRoute from './api/project'
|
||||
import providerRoute from './api/provider'
|
||||
import publicInviteRoute from './api/public-invite'
|
||||
import publicShareRoute from './api/public-share'
|
||||
import s3Route from './api/s3'
|
||||
import shareRoute from './api/share'
|
||||
import skillRoute from './api/skill'
|
||||
import teamRoute from './api/team'
|
||||
import uploadRoute from './api/upload'
|
||||
import versionStackRoute from './api/versionStack'
|
||||
import agentRoute from './agent'
|
||||
import attachmentRoute from './attachment'
|
||||
import authnRoute from './authn'
|
||||
import collectionRoute from './collection'
|
||||
import fileRoute from './file'
|
||||
import folderRoute from './folder'
|
||||
import inviteRoute from './invite'
|
||||
import metadataRoute from './metadata'
|
||||
import { authMiddleware } from './middleware/auth'
|
||||
import { tokenAuthMiddleware } from './middleware/tokenAuth'
|
||||
import notificationRoute from './notification'
|
||||
import projectRoute from './project'
|
||||
import providerRoute from './provider'
|
||||
import publicInviteRoute from './public-invite'
|
||||
import publicShareRoute from './public-share'
|
||||
import s3Route from './s3'
|
||||
import shareRoute from './share'
|
||||
import skillRoute from './skill'
|
||||
import teamRoute from './team'
|
||||
import uploadRoute, { localUploadRoute } from './upload'
|
||||
import versionStackRoute from './versionStack'
|
||||
|
||||
type User = Prisma.UserGetPayload<Record<string, never>>
|
||||
|
||||
@@ -30,6 +31,15 @@ const apiRoute = new Hono<{ Variables: { user: User } }>()
|
||||
.route('/', authnRoute)
|
||||
.route('/', publicInviteRoute)
|
||||
.route('/', publicShareRoute)
|
||||
.route('/', localUploadRoute)
|
||||
|
||||
// Selected routes supporting API token authentication
|
||||
.use('/projects', tokenAuthMiddleware)
|
||||
.use('/projects/*', tokenAuthMiddleware)
|
||||
.use('/folders', tokenAuthMiddleware)
|
||||
.use('/folders/*', tokenAuthMiddleware)
|
||||
.use('/teams/:teamId/upload/tasks', tokenAuthMiddleware)
|
||||
.use('/teams/:teamId/upload/tasks/*', tokenAuthMiddleware)
|
||||
|
||||
// Protected Routes
|
||||
.use('*', authMiddleware)
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { describe, expect, it, vi, beforeEach } from 'vitest'
|
||||
import { Hono } from 'hono'
|
||||
import { authMiddleware } from './auth'
|
||||
import { auth } from '@shumai/core/src/auth/auth'
|
||||
import { prisma } from '@shumai/db'
|
||||
|
||||
vi.mock('@shumai/core/src/auth/auth', () => ({
|
||||
auth: {
|
||||
api: {
|
||||
getSession: vi.fn(),
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@shumai/db', () => ({
|
||||
prisma: {
|
||||
user: {
|
||||
findUnique: vi.fn(),
|
||||
},
|
||||
teamMember: {
|
||||
findFirst: vi.fn(),
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
describe('authMiddleware', () => {
|
||||
const app = new Hono()
|
||||
.use('*', authMiddleware)
|
||||
.get('/test', (c) => c.text('ok'))
|
||||
.post('/test', (c) => c.text('ok'))
|
||||
.post('/test/search', (c) => c.text('ok'))
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks()
|
||||
delete process.env.SHUMAI_DEMO_MODE
|
||||
})
|
||||
|
||||
it('allows access when authenticated', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue({ user: { id: 'user1' } } as any)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user1' } as any)
|
||||
|
||||
const res = await app.request('/test')
|
||||
expect(res.status).toBe(200)
|
||||
})
|
||||
|
||||
it('rejects when not authenticated', async () => {
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue(null)
|
||||
|
||||
const res = await app.request('/test')
|
||||
expect(res.status).toBe(401)
|
||||
})
|
||||
|
||||
describe('read-only mode', () => {
|
||||
beforeEach(() => {
|
||||
process.env.SHUMAI_DEMO_MODE = '1'
|
||||
})
|
||||
|
||||
it('allows GET for reviewer', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue({ user: { id: 'user1' } } as any)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user1' } as any)
|
||||
|
||||
const res = await app.request('/test')
|
||||
expect(res.status).toBe(200)
|
||||
})
|
||||
|
||||
it('rejects POST for reviewer', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue({ user: { id: 'user1' } } as any)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user1' } as any)
|
||||
vi.mocked(prisma.teamMember.findFirst).mockResolvedValue(null) // No owner/editor role
|
||||
|
||||
const res = await app.request('/test', { method: 'POST' })
|
||||
expect(res.status).toBe(403)
|
||||
expect(await res.json()).toEqual({ error: 'System is in read-only mode' })
|
||||
})
|
||||
|
||||
it('allows POST /search for reviewer', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue({ user: { id: 'user1' } } as any)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user1' } as any)
|
||||
|
||||
const res = await app.request('/test/search', { method: 'POST' })
|
||||
expect(res.status).toBe(200)
|
||||
})
|
||||
|
||||
it('allows POST for owner', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue({ user: { id: 'user1' } } as any)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user1' } as any)
|
||||
vi.mocked(prisma.teamMember.findFirst).mockResolvedValue({
|
||||
id: 'member1',
|
||||
role: 'owner',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
const res = await app.request('/test', { method: 'POST' })
|
||||
expect(res.status).toBe(200)
|
||||
})
|
||||
|
||||
it('allows POST for editor', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(auth.api.getSession).mockResolvedValue({ user: { id: 'user1' } } as any)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user1' } as any)
|
||||
vi.mocked(prisma.teamMember.findFirst).mockResolvedValue({
|
||||
id: 'member1',
|
||||
role: 'editor',
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any)
|
||||
|
||||
const res = await app.request('/test', { method: 'POST' })
|
||||
expect(res.status).toBe(200)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -10,6 +10,11 @@ export const authMiddleware = createMiddleware<{
|
||||
user: User
|
||||
}
|
||||
}>(async (c, next) => {
|
||||
if (c.get('user')) {
|
||||
await next()
|
||||
return
|
||||
}
|
||||
|
||||
const session = await auth.api.getSession({
|
||||
headers: c.req.raw.headers,
|
||||
})
|
||||
@@ -28,6 +33,24 @@ export const authMiddleware = createMiddleware<{
|
||||
}
|
||||
|
||||
c.set('user', user)
|
||||
|
||||
if (process.env.SHUMAI_DEMO_MODE === '1') {
|
||||
const method = c.req.method.toUpperCase()
|
||||
const path = c.req.path
|
||||
if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(method) && !path.endsWith('/search')) {
|
||||
const member = await prisma.teamMember.findFirst({
|
||||
where: {
|
||||
userId: user.id,
|
||||
role: { in: ['owner', 'editor'] },
|
||||
},
|
||||
})
|
||||
|
||||
if (!member) {
|
||||
return c.json({ error: 'System is in read-only mode' }, 403)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await next()
|
||||
} catch {
|
||||
return c.json({ error: 'Authentication failed' }, 401)
|
||||