mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-21 11:48:40 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e33cbdb37c | ||
|
|
0514a19702 | ||
|
|
9fb9e608b9 | ||
|
|
cbb25b49e7 | ||
|
|
ce8fc5d128 | ||
|
|
34bacf5421 | ||
|
|
98bbfa51ae | ||
|
|
fc9dd8a5f7 | ||
|
|
c7fef79860 | ||
|
|
efa0e75a06 | ||
|
|
b5ff76960f | ||
|
|
ca0c2ece75 | ||
|
|
c8b1d521d8 | ||
|
|
4bd5083ac1 | ||
|
|
c6527650d0 | ||
|
|
d4a1eff715 | ||
|
|
463cd61cfd | ||
|
|
93b4656939 | ||
|
|
3e122a86f0 | ||
|
|
292307e28e | ||
|
|
982bca9383 | ||
|
|
d1061a1023 | ||
|
|
994f1d3b44 | ||
|
|
e057f17f8e | ||
|
|
66d07d1fb0 | ||
|
|
5542538a64 |
@@ -46,3 +46,12 @@ jobs:
|
||||
- name: Test Local Release
|
||||
run: |
|
||||
bun run release:local --skip-check --out /tmp/shumai-local-release --force
|
||||
|
||||
echo "Smoke testing shumai..."
|
||||
bun run /tmp/shumai-local-release/node/node_modules/@shumai-one/shumai/bin/shumai-app.js --check
|
||||
|
||||
echo "Smoke testing shumai-agent..."
|
||||
bun run /tmp/shumai-local-release/node/node_modules/@shumai-one/shumai-agent/bin/shumai-agent-app.js --check
|
||||
|
||||
echo "Smoke testing shumai-transcode..."
|
||||
bun run /tmp/shumai-local-release/node/node_modules/@shumai-one/shumai-transcode/bin/shumai-transcode-app.js --check
|
||||
|
||||
@@ -16,48 +16,54 @@
|
||||
|
||||
## Features
|
||||
|
||||
- **S3-Compatible & Local FS Storage**: Securely store and serve your creative assets using local filesystems or any S3-compatible cloud storage (AWS S3, Cloudflare R2, MinIO, etc.).
|
||||
- **Comments with Draw Annotations & Timestamps**: Give precise feedback with frame-specific drawing tools and exact timestamps directly on video and image assets.
|
||||
- **Easy Share Links & Collections**: Create secure external share links and curated media collections to collaborate with external clients and stakeholders.
|
||||
- **Team & Project Member Management**: Manage granular workspace permissions using team-level and project-level role-based access control.
|
||||
- **Scalable Transcode Workflows via Temporal**: Offload resource-heavy video transcoding to a reliable background worker pool orchestrated by Temporal.
|
||||
- **Flexible Asset Metadata System**: Customize your workspace metadata with dynamic, user-defined fields tailored to your production pipeline.
|
||||
- **S3-Compatible & Local Storage**: Securely store and serve your creative assets using a local filesystem or any S3-compatible cloud storage (AWS S3, Cloudflare R2, MinIO, etc.).
|
||||
- **Frame-by-Frame Annotations & Comments**: Give precise feedback using frame-specific drawing tools and timestamped comments directly on video and image assets.
|
||||
- **Secure Sharing & Collections**: Create secure public share links and curated media collections to collaborate with clients and stakeholders.
|
||||
- **Granular Access Control**: Manage workspace permissions using team-level and project-level role-based access controls.
|
||||
- **Distributed Transcoding via Temporal**: Offload resource-heavy video transcoding to a background worker pool orchestrated by Temporal.
|
||||
- **Custom Asset Metadata**: Define and customize dynamic metadata fields tailored to your production pipeline.
|
||||
|
||||
#### Shumai Agent
|
||||
|
||||
- **Chat with Agent as a Collaborator**: Converse with a context-aware AI agent directly within your project workspace.
|
||||
- **Extend Agent with Skills**: Easily register new tools and custom skills for the agent to run and automate workflows.
|
||||
- **Secure Sandboxed Execution**: Execute agent scripts and tasks in a secure, isolated sandbox environment.
|
||||
- **Collaborative AI Chat**: Converse with a context-aware AI agent directly within your project workspace.
|
||||
- **Custom Skills & Tools**: Extend the agent's capabilities by registering custom scripts, tools, and automation skills.
|
||||
- **Isolated Sandbox Execution**: Run agent-submitted scripts securely within a sandboxed environment.
|
||||
- **AI-Powered Metadata Autofill**: Automatically generate tags, descriptions, and custom metadata for new assets using Google Gemini.
|
||||
- **Semantic Asset Search**: Find assets instantly based on their visual and textual content using vector embeddings.
|
||||
- **Semantic Search**: Locate assets instantly based on visual or conceptual search queries using vector embeddings.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### Option 1: Docker Compose (Quick Start)
|
||||
Below is a quickstart guide for running Shumai with local storage. For advanced configuration options (including S3-compatible storage and Temporal workflow orchestration), see our [Documentation](https://docs.shumai.one/getting-started/overview).
|
||||
|
||||
The easiest way to run Shumai is using Docker Compose. You do not need to clone the repository:
|
||||
### Option 1: Docker Compose
|
||||
|
||||
1. Create a `docker-compose.yaml` file by copying the configuration from [GitHub](https://github.com/shumaiOne/shumai/blob/main/docker-compose/local/docker-compose.yaml) or download it directly:
|
||||
Docker Compose is the fastest way to get Shumai running. You do not need to clone the repository or install packages manually. Ensure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed, then follow these steps:
|
||||
|
||||
1. Create and navigate to a new directory for your configuration and data volumes:
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
2. Download the `docker-compose.yaml` file:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/local/docker-compose.yaml
|
||||
```
|
||||
2. Configure environment variables (if deploying remotely):
|
||||
If you are deploying Shumai to a remote server (e.g., AWS ECS, EC2, VPS), you must edit `docker-compose.yaml`, set `AWS_ENDPOINT_URL_S3` to your server's public IP or domain name (e.g., `http://12.345.567.789`).
|
||||
3. Start the services:
|
||||
3. **Configure Environment Variables (Remote Deployments only):**
|
||||
If deploying Shumai to a remote server (e.g., AWS EC2, VPS), edit docker-compose.yaml and add AWS_ENDPOINT_URL_S3 under the environment section, setting it to your server's public IP address or domain name (e.g., http://12.34.56.78).
|
||||
4. Start the services in detached mode:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
4. Access the application in your browser at `http://localhost:3000` (or `http://<your-server-ip>:3000` for remote deployments).
|
||||
5. Open your browser and access Shumai at `http://localhost:3000` (or `http://<your-server-ip>:3000` for remote deployments).
|
||||
|
||||
### Option 2: Install from NPM
|
||||
### Option 2: Install via NPM / Package Manager
|
||||
|
||||
You can run Shumai using the npm package `@shumai-one/shumai`. This option allows you to install and run Shumai globally or locally via your favorite package manager.
|
||||
Shumai is published as `@shumai-one/shumai` on NPM. This option allows you to run Shumai globally or locally.
|
||||
|
||||
#### Step 1: Start PostgreSQL with pgvector
|
||||
|
||||
Shumai requires PostgreSQL with the `pgvector` extension. You can spin up a pre-configured database container using Docker:
|
||||
Shumai requires PostgreSQL with the `pgvector` extension. Start a pre-configured database container using Docker:
|
||||
|
||||
```bash
|
||||
docker run --name shumai_postgres \
|
||||
@@ -68,35 +74,52 @@ docker run --name shumai_postgres \
|
||||
-d pgvector/pgvector:pg18
|
||||
```
|
||||
|
||||
#### Step 2: Install Shumai globally
|
||||
#### Step 2: Create a workspace folder
|
||||
|
||||
Install the main Shumai package using `npm`, `pnpm`, `bun`, or `yarn`:
|
||||
Create a dedicated directory to store your environment configuration and media files (which are saved in a `./data` directory by default):
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Prerequisites**:
|
||||
```bash
|
||||
mkdir shumai && cd shumai
|
||||
```
|
||||
|
||||
#### Step 3: Install Platform-Specific Dependencies
|
||||
|
||||
Make sure the following system dependencies are installed on your host machine before installing Shumai:
|
||||
|
||||
| Package | Description | Ubuntu/Debian | Fedora | Arch | macOS |
|
||||
| :------------- | :------------------------------------------------------ | :------------------------------- | :------------------------------- | :-------------------------------------- | :--------------------- |
|
||||
| **ffmpeg** | Media transcoding and metadata extraction | `sudo apt install -y ffmpeg` | `sudo dnf install -y ffmpeg` | `sudo pacman -S --noconfirm ffmpeg` | `brew install ffmpeg` |
|
||||
| **bubblewrap** | Sandboxing environment for secure AI agent execution | `sudo apt install -y bubblewrap` | `sudo dnf install -y bubblewrap` | `sudo pacman -S --noconfirm bubblewrap` | _NOT REQUIRED_ |
|
||||
| **socat** | Bidirectional socket relay for sandbox network bridging | `sudo apt install -y socat` | `sudo dnf install -y socat` | `sudo pacman -S --noconfirm socat` | `brew install socat` |
|
||||
| **ripgrep** | Fast search tool for workspace security policies | `sudo apt install -y ripgrep` | `sudo dnf install -y ripgrep` | `sudo pacman -S --noconfirm ripgrep` | `brew install ripgrep` |
|
||||
|
||||
> [!NOTE]
|
||||
> **Ubuntu 24.04+ Note:** These releases restrict unprivileged user namespaces by default. To allow `bubblewrap` and the sandbox isolation layer to function, disable this restriction:
|
||||
>
|
||||
> - **Agent Sandbox**: Because Shumai uses a secure sandboxed execution environment for its AI agents, your machine must meet the platform requirements for the sandbox. Please verify you have installed the required dependencies for your OS as detailed in the [Agent Sandbox Platform Support](#agent-sandbox-platform-support) section.
|
||||
> - **FFmpeg & FFprobe**: Shumai requires `ffmpeg` and `ffprobe` to be installed on your system path for handling media transcoding and metadata extraction.
|
||||
> - macOS: `brew install ffmpeg`
|
||||
> - Ubuntu/Debian: `sudo apt-get install ffmpeg`
|
||||
> - Fedora: `sudo dnf install ffmpeg`
|
||||
> ```bash
|
||||
> sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
> ```
|
||||
>
|
||||
> Alternatively, configure an AppArmor profile that grants user namespace creation (`userns`) privileges to the relevant binaries.
|
||||
|
||||
#### Step 4: Install Shumai globally
|
||||
|
||||
Install Shumai globally using your preferred package manager:
|
||||
|
||||
```bash
|
||||
# NPM
|
||||
npm install -g @shumai-one/shumai
|
||||
# or: pnpm add -g @shumai-one/shumai
|
||||
# or: bun add -g @shumai-one/shumai
|
||||
|
||||
# PNPM
|
||||
pnpm add -g @shumai-one/shumai
|
||||
|
||||
# Bun
|
||||
bun add -g @shumai-one/shumai
|
||||
```
|
||||
|
||||
#### Step 3: Configure Environment Variables
|
||||
#### Step 5: Configure Environment Variables
|
||||
|
||||
Create a new directory for your Shumai configuration and navigate into it:
|
||||
|
||||
```bash
|
||||
mkdir shumai
|
||||
cd shumai
|
||||
```
|
||||
|
||||
Create a `.env` file in this folder and add the following configuration (which aligns with the Docker database command in Step 1):
|
||||
Create a `.env` file in your workspace folder (`shumai/`) and add the following configuration:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@localhost:5432/shumai_db?schema=public
|
||||
@@ -107,23 +130,42 @@ AWS_ENDPOINT_URL_S3=http://localhost
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> If you are deploying on a remote server, make sure to change `AWS_ENDPOINT_URL_S3` from `http://localhost` to your server's public IP address or domain name. `SHUMAI_SERVER_PORT` defaults to `3000` if not set.
|
||||
> If deploying on a remote server, update `AWS_ENDPOINT_URL_S3` from `http://localhost` to your server's public IP address or domain name.
|
||||
|
||||
#### Step 4: Run Shumai
|
||||
#### Step 6: Run Shumai
|
||||
|
||||
Start the application from your configuration folder:
|
||||
Start the application from your workspace folder:
|
||||
|
||||
```bash
|
||||
shumai
|
||||
```
|
||||
|
||||
On startup, Shumai will automatically run the necessary database migrations and start the web server at `http://localhost:3000`.
|
||||
Alternatively, you can run and manage Shumai in daemon mode:
|
||||
|
||||
- **Start in daemon mode**:
|
||||
```bash
|
||||
shumai -d
|
||||
```
|
||||
- **Stop Shumai**:
|
||||
```bash
|
||||
shumai stop
|
||||
```
|
||||
- **Restart Shumai**:
|
||||
```bash
|
||||
shumai restart
|
||||
```
|
||||
- **Show/tail logs**:
|
||||
```bash
|
||||
shumai logs
|
||||
```
|
||||
|
||||
On startup, Shumai will automatically run database migrations and start the web server at `http://localhost:3000`.
|
||||
|
||||
---
|
||||
|
||||
### Option 3: Run from Source
|
||||
### Option 3: Run from Source (Development)
|
||||
|
||||
To run Shumai locally for development:
|
||||
To set up Shumai locally for development:
|
||||
|
||||
1. Clone the repository and install dependencies:
|
||||
```bash
|
||||
@@ -131,151 +173,15 @@ To run Shumai locally for development:
|
||||
cd shumai
|
||||
bun install
|
||||
```
|
||||
2. Start the `pgvector` database container (as described in Option 2, Step 1).
|
||||
3. Create a `.env` file at the root of the workspace using the example configuration from Option 2, Step 3.
|
||||
4. Run the database migrations:
|
||||
2. Start the `pgvector` database container (as described in **Option 2, Step 1**).
|
||||
3. Create a `.env` file at the root of the workspace using the configuration from **Option 2, Step 5**.
|
||||
4. Apply the database schema migrations:
|
||||
```bash
|
||||
bun run db:migrate
|
||||
```
|
||||
5. Start the development server:
|
||||
5. Start the local development server:
|
||||
```bash
|
||||
bun run dev
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Advanced: Distributed Execution with Temporal
|
||||
|
||||
For production environments or workloads with heavy media processing and AI agent requirements, Shumai can delegate background activities to **Temporal**. This provides distributed queue management, task retries, and high availability.
|
||||
|
||||
### How it works
|
||||
|
||||
By setting the environment variable `WORKFLOW_EXECUTOR=temporal`, the main Shumai application will delegate background workflows to a Temporal cluster instead of using the local polling-based engine. This allows you to run independent worker processes (`shumai-agent` and `shumai-transcode`) on separate specialized machines (e.g., transcoding workers on GPU-accelerated instances, and agents on sandboxed environments).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Storage Requirement**: Because the main Shumai application and the background workers run on different machines in a distributed setup, **you must use S3-compatible storage** (e.g., AWS S3, Cloudflare R2, MinIO). Local storage (`STORAGE_BACKEND=local`) will not work because the workers and the main application do not share a common local filesystem.
|
||||
|
||||
### 1. Set up the Core Stack
|
||||
|
||||
Spin up the database, Temporal server, and the main Shumai web application on your primary host machine using Docker Compose:
|
||||
|
||||
1. Download the Temporal-enabled `docker-compose.yaml` file:
|
||||
```bash
|
||||
curl -o docker-compose.yaml https://raw.githubusercontent.com/shumaiOne/shumai/main/docker-compose/temporal/docker-compose.yaml
|
||||
```
|
||||
2. Configure the environment variables in `docker-compose.yaml`:
|
||||
- Set the S3 storage environment variables (`AWS_ENDPOINT_URL_S3`, `S3_REGION`, `S3_BUCKET`, etc.).
|
||||
- If deploying to a remote host (e.g., AWS ECS, EC2, VPS), update `AWS_ENDPOINT_URL_S3` under the `shumai` service to your server's public IP address or domain name.
|
||||
- Note: The `TEMPORAL_ADDRESS` environment variable for the main Shumai application is pre-configured as `temporal:7233` (referencing the Temporal container within the same Docker network) and does not need to be changed.
|
||||
3. Start the services:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
This exposes the Shumai Web UI at `http://<host-ip>:3000`, the database at port `5432`, the Temporal gRPC service at port `7233`, and the Temporal dashboard at `http://<host-ip>:8080`. Ensure that port `7233` is exposed and accessible by your background worker machines.
|
||||
|
||||
### 2. Run the Background Workers
|
||||
|
||||
When running in Temporal mode, the main Shumai application only submits tasks to the Temporal queue. To process these tasks, you must run at least one agent worker and one transcoding worker on their respective machines.
|
||||
|
||||
> [!NOTE]
|
||||
> **Network & Access Requirements**:
|
||||
>
|
||||
> - Both the transcode worker and the agent worker must have network access to the PostgreSQL database and the S3-compatible storage.
|
||||
> - Because the transcode worker needs to download and upload large media files for transcoding, it is highly recommended to deploy the transcode worker in the same cloud provider and region as your S3 storage to minimize transfer latency and data egress fees.
|
||||
|
||||
Configure a `.env` file on each worker machine. It must contain the following variables:
|
||||
|
||||
```env
|
||||
DATABASE_URL=postgresql://shumai:shumai_password@<host-ip>:5432/shumai_db?schema=public
|
||||
WORKFLOW_EXECUTOR=temporal
|
||||
TEMPORAL_ADDRESS=<host-ip>:7233
|
||||
STORAGE_BACKEND=s3
|
||||
AWS_ENDPOINT_URL_S3=http://<host-ip> # Or your public S3/R2 endpoint URL
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=shumai
|
||||
S3_ACCESS_KEY_ID=your-access-key-id
|
||||
S3_SECRET_ACCESS_KEY=your-secret-access-key
|
||||
```
|
||||
|
||||
Run the workers:
|
||||
|
||||
- **Transcoding Worker**:
|
||||
Install `@shumai-one/shumai-transcode` and run it on a machine with sufficient transcoding resources:
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **FFmpeg Requirement**: The transcoding worker requires `ffmpeg` and `ffprobe` to be installed and available on the machine's system path.
|
||||
>
|
||||
> - macOS: `brew install ffmpeg`
|
||||
> - Ubuntu/Debian: `sudo apt-get install ffmpeg`
|
||||
|
||||
```bash
|
||||
npm install -g @shumai-one/shumai-transcode
|
||||
# Run worker with the configured .env
|
||||
shumai-transcode
|
||||
```
|
||||
|
||||
- **AI Agent Worker**:
|
||||
Install `@shumai-one/shumai-agent` and run it on a machine configured for Agent tasks:
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Because the AI Agent worker runs sandboxed agent scripts, the host machine must have the necessary dependencies installed. Please refer to [Agent Sandbox Platform Support](#agent-sandbox-platform-support) for setup details.
|
||||
|
||||
```bash
|
||||
npm install -g @shumai-one/shumai-agent
|
||||
# Run worker with the configured .env (must include GEMINI_API_KEY)
|
||||
shumai-agent
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Agent Sandbox Platform Support
|
||||
|
||||
Shumai uses `anthropic-experimental/sandbox-runtime` to execute AI agent scripts and tasks in a secure, isolated sandbox environment. When installing Shumai or the AI Agent worker from NPM, ensure your host platform meets the following requirements:
|
||||
|
||||
### Platform Support
|
||||
|
||||
- **macOS**: Uses `sandbox-exec` with custom profiles (no additional OS-level containerization dependencies).
|
||||
- **Linux**: Uses `bubblewrap` (bwrap) for containerization.
|
||||
- **Windows**: Not yet supported.
|
||||
|
||||
### Platform-Specific Dependencies
|
||||
|
||||
#### Linux
|
||||
|
||||
The following system packages must be installed on your Linux host:
|
||||
|
||||
- `bubblewrap` - Container runtime
|
||||
- Ubuntu/Debian: `sudo apt-get install bubblewrap`
|
||||
- Fedora: `sudo dnf install bubblewrap`
|
||||
- Arch: `sudo pacman -S bubblewrap`
|
||||
- `socat` - Socket relay for proxy bridging
|
||||
- Ubuntu/Debian: `sudo apt-get install socat`
|
||||
- Fedora: `sudo dnf install socat`
|
||||
- Arch: `sudo pacman -S socat`
|
||||
- `ripgrep` - Fast search tool for deny path detection
|
||||
- Ubuntu/Debian: `sudo apt-get install ripgrep`
|
||||
- Fedora: `sudo dnf install ripgrep`
|
||||
- Arch: `sudo pacman -S ripgrep`
|
||||
|
||||
**Ubuntu 24.04+ Note**: These releases restrict unprivileged user namespaces by default. Disable this restriction so that `bubblewrap` and the seccomp isolation layer can function:
|
||||
|
||||
```bash
|
||||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
```
|
||||
|
||||
**Optional Linux Dependencies (for seccomp fallback)**:
|
||||
Pre-generated seccomp filters are provided for x86-64 and arm architectures. If you run on a different architecture, you will need a C compiler and development files:
|
||||
|
||||
- `gcc` or `clang`
|
||||
- `libseccomp` development headers
|
||||
- Ubuntu/Debian: `sudo apt-get install gcc libseccomp-dev`
|
||||
- Fedora: `sudo dnf install gcc libseccomp-devel`
|
||||
- Arch: `sudo pacman -S gcc libseccomp`
|
||||
|
||||
#### macOS
|
||||
|
||||
The following dependencies are required on macOS:
|
||||
|
||||
- `ripgrep` - Fast search tool for deny path detection
|
||||
- Install via Homebrew: `brew install ripgrep`
|
||||
- Or download from: https://github.com/BurntSushi/ripgrep/releases
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -4,6 +4,12 @@ loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { TaskQueueAgent } from '@shumai/workflow-core'
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Agent worker evaluated successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
async function run() {
|
||||
// Initialize workflows and activities
|
||||
@@ -23,7 +29,7 @@ async function run() {
|
||||
await workflowService.startWorkers(TaskQueueAgent, options)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
handleDaemonCommands('shumai-agent', run).catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -3,6 +3,12 @@ loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { TaskQueueTranscode, workflowService } from '@shumai/workflow-core'
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Transcode worker evaluated successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
async function run() {
|
||||
// Initialize workflows and activities
|
||||
@@ -22,7 +28,7 @@ async function run() {
|
||||
await workflowService.startWorkers(TaskQueueTranscode, options)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
handleDaemonCommands('shumai-transcode', run).catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
+121
-108
@@ -1,123 +1,136 @@
|
||||
import { join } from 'node:path'
|
||||
import { loadEnvConfig } from '@shumai/core/src/env-loader'
|
||||
import { join } from 'node:path'
|
||||
loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
|
||||
import index from '@shumai/webui/index.html'
|
||||
|
||||
import { initAgentWorkflows } from '@shumai/agent'
|
||||
import { app } from '@shumai/api'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { app } from '@shumai/api'
|
||||
|
||||
// Initialize workflows and activities for local executor mode
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
workflowService.start()
|
||||
if (process.env.WORKFLOW_EXECUTOR === 'temporal') {
|
||||
const args = process.argv.slice(2)
|
||||
let workersOption = ''
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--workers') {
|
||||
workersOption = args[i + 1] || ''
|
||||
} else if (args[i].startsWith('--workers=')) {
|
||||
workersOption = args[i].split('=')[1]
|
||||
}
|
||||
}
|
||||
|
||||
const queuesToStart: string[] = []
|
||||
if (workersOption === 'agent' || workersOption === 'ai') {
|
||||
queuesToStart.push('agent_queue')
|
||||
} else if (workersOption === 'transcode') {
|
||||
queuesToStart.push('transcode_queue')
|
||||
} else if (workersOption === 'all') {
|
||||
queuesToStart.push('agent_queue', 'transcode_queue')
|
||||
}
|
||||
|
||||
Promise.all(queuesToStart.map((q) => workflowService.startWorkers(q))).catch(console.error)
|
||||
}
|
||||
|
||||
const isProd = process.env.NODE_ENV === 'production'
|
||||
|
||||
const port = process.env.SHUMAI_SERVER_PORT ? parseInt(process.env.SHUMAI_SERVER_PORT) : 3000
|
||||
|
||||
const server = Bun.serve(
|
||||
isProd
|
||||
? {
|
||||
port,
|
||||
maxRequestBodySize: process.env.MAX_REQUEST_BODY_SIZE
|
||||
? parseInt(process.env.MAX_REQUEST_BODY_SIZE)
|
||||
: 1024 * 1024 * 1024 * 10, // Default 10GB
|
||||
development: false,
|
||||
async fetch(req) {
|
||||
const url = new URL(req.url)
|
||||
|
||||
if (!url.pathname.startsWith('/api/') && !url.pathname.startsWith('/files/')) {
|
||||
const filepath = join(import.meta.dir, url.pathname)
|
||||
const file = Bun.file(filepath)
|
||||
if (await file.exists()) {
|
||||
try {
|
||||
const stat = await file.stat()
|
||||
if (stat.isFile()) {
|
||||
return new Response(file)
|
||||
}
|
||||
} catch {
|
||||
// ignore stat errors
|
||||
}
|
||||
}
|
||||
|
||||
const htmlFile = Bun.file(join(import.meta.dir, 'index.html'))
|
||||
if (await htmlFile.exists()) {
|
||||
return new Response(htmlFile, {
|
||||
headers: { 'Content-Type': 'text/html' },
|
||||
})
|
||||
}
|
||||
|
||||
const htmlFileNpm = Bun.file(join(import.meta.dir, 'shumai-app.html'))
|
||||
if (await htmlFileNpm.exists()) {
|
||||
return new Response(htmlFileNpm, {
|
||||
headers: { 'Content-Type': 'text/html' },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return app.fetch(req)
|
||||
},
|
||||
}
|
||||
: {
|
||||
port,
|
||||
maxRequestBodySize: process.env.MAX_REQUEST_BODY_SIZE
|
||||
? parseInt(process.env.MAX_REQUEST_BODY_SIZE)
|
||||
: 1024 * 1024 * 1024 * 10, // Default 10GB
|
||||
development: true,
|
||||
fetch: app.fetch,
|
||||
routes: {
|
||||
// Serve index.html for root
|
||||
'/': index,
|
||||
|
||||
// Proxy API requests to Hono
|
||||
'/api/*': app.fetch,
|
||||
'/files/*': app.fetch,
|
||||
|
||||
// Catch-all for SPA routing (fallback to index.html)
|
||||
'/*': index,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
console.log(`🚀 Server running at ${server.url}`)
|
||||
|
||||
const shutdown = () => {
|
||||
console.log('\nShutting down gracefully...')
|
||||
assetService.stopCleanupJob()
|
||||
server.stop(true)
|
||||
if (process.argv.includes('--check')) {
|
||||
console.log('✅ Web app evaluated successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
process.on('SIGINT', shutdown)
|
||||
process.on('SIGTERM', shutdown)
|
||||
async function run() {
|
||||
// Initialize workflows and activities for local executor mode
|
||||
initAgentWorkflows()
|
||||
initTranscodeWorkflows()
|
||||
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
workflowService.start()
|
||||
if (process.env.WORKFLOW_EXECUTOR === 'temporal') {
|
||||
const args = process.argv.slice(2)
|
||||
let workersOption = ''
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--workers') {
|
||||
workersOption = args[i + 1] || ''
|
||||
} else if (args[i].startsWith('--workers=')) {
|
||||
workersOption = args[i].split('=')[1]
|
||||
}
|
||||
}
|
||||
|
||||
const queuesToStart: string[] = []
|
||||
if (workersOption === 'agent' || workersOption === 'ai') {
|
||||
queuesToStart.push('agent_queue')
|
||||
} else if (workersOption === 'transcode') {
|
||||
queuesToStart.push('transcode_queue')
|
||||
} else if (workersOption === 'all') {
|
||||
queuesToStart.push('agent_queue', 'transcode_queue')
|
||||
}
|
||||
|
||||
Promise.all(queuesToStart.map((q) => workflowService.startWorkers(q))).catch(console.error)
|
||||
}
|
||||
|
||||
const isProd = process.env.NODE_ENV === 'production'
|
||||
|
||||
const port = process.env.SHUMAI_SERVER_PORT ? parseInt(process.env.SHUMAI_SERVER_PORT) : 3000
|
||||
|
||||
interface HtmlBundleFile {
|
||||
input: string
|
||||
path: string
|
||||
loader: string
|
||||
isEntry: boolean
|
||||
headers: Record<string, string>
|
||||
}
|
||||
|
||||
interface HtmlBundle {
|
||||
index: string
|
||||
files: HtmlBundleFile[]
|
||||
}
|
||||
|
||||
// Bun's native Serve config routes require specific handler types that are hard
|
||||
// to dynamically construct without using 'any'.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const routes: Record<string, any> = {
|
||||
'/api/*': app.fetch,
|
||||
'/files/*': app.fetch,
|
||||
}
|
||||
|
||||
if (isProd) {
|
||||
const bundle = index as unknown as HtmlBundle
|
||||
if (bundle && typeof bundle === 'object' && 'files' in bundle && Array.isArray(bundle.files)) {
|
||||
for (const file of bundle.files) {
|
||||
const routePath = file.path.startsWith('.') ? file.path.slice(1) : file.path
|
||||
const filePath = join(import.meta.dir, file.path)
|
||||
routes[routePath] = () => new Response(Bun.file(filePath), { headers: file.headers })
|
||||
}
|
||||
|
||||
const mainHtmlPath = join(import.meta.dir, bundle.index)
|
||||
const mainHtmlFile = Bun.file(mainHtmlPath)
|
||||
const mainHtmlHeaders = bundle.files.find((f) => f.path === bundle.index)?.headers || {
|
||||
'content-type': 'text/html;charset=utf-8',
|
||||
}
|
||||
const serveMainHtml = () => new Response(mainHtmlFile, { headers: mainHtmlHeaders })
|
||||
|
||||
routes['/'] = serveMainHtml
|
||||
routes['/*'] = serveMainHtml
|
||||
} else {
|
||||
const fallbackHtmlPath = join(import.meta.dir, 'shumai-app.html')
|
||||
const serveFallbackHtml = () =>
|
||||
new Response(Bun.file(fallbackHtmlPath), {
|
||||
headers: { 'content-type': 'text/html;charset=utf-8' },
|
||||
})
|
||||
routes['/'] = serveFallbackHtml
|
||||
routes['/*'] = serveFallbackHtml
|
||||
}
|
||||
} else {
|
||||
routes['/'] = index
|
||||
routes['/*'] = index
|
||||
}
|
||||
|
||||
const server = Bun.serve({
|
||||
port,
|
||||
maxRequestBodySize: process.env.MAX_REQUEST_BODY_SIZE
|
||||
? parseInt(process.env.MAX_REQUEST_BODY_SIZE)
|
||||
: 1024 * 1024 * 1024 * 10, // Default 10GB
|
||||
development: !isProd,
|
||||
fetch: app.fetch,
|
||||
routes,
|
||||
})
|
||||
|
||||
console.log(`🚀 Server running at ${server.url}`)
|
||||
|
||||
const shutdown = () => {
|
||||
console.log('\nShutting down gracefully...')
|
||||
assetService.stopCleanupJob()
|
||||
server.stop(true)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
process.on('SIGINT', shutdown)
|
||||
process.on('SIGTERM', shutdown)
|
||||
}
|
||||
|
||||
handleDaemonCommands('shumai', run).catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
@@ -22,7 +22,7 @@ services:
|
||||
ports:
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- ../../data:/app/data
|
||||
- ./data:/app/data
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_URL: postgres://shumai:shumai_password@shumai_postgres:5432/shumai_db?search_path=public&sslmode=disable
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/core",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, expect, it, vi, beforeEach, afterEach, type MockInstance } from 'vitest'
|
||||
import { existsSync, writeFileSync, readFileSync, rmSync, mkdirSync, openSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { handleDaemonCommands } from './daemon'
|
||||
import { spawn } from 'node:child_process'
|
||||
|
||||
// Mock os.homedir to return a temporary directory for test isolation
|
||||
vi.mock('node:os', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('node:os')>()
|
||||
const mockTempDir = `${actual.tmpdir()}/shumai-test-daemon-${Math.random().toString(36).substring(2)}`
|
||||
return {
|
||||
...actual,
|
||||
homedir: () => mockTempDir,
|
||||
}
|
||||
})
|
||||
|
||||
// Mock node:child_process for spawn tests
|
||||
vi.mock('node:child_process', () => {
|
||||
return {
|
||||
spawn: vi.fn(() => {
|
||||
return {
|
||||
pid: 99999,
|
||||
unref: vi.fn(),
|
||||
}
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
// Mock node:fs to mock openSync for ESM safety
|
||||
vi.mock('node:fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('node:fs')>()
|
||||
return {
|
||||
...actual,
|
||||
openSync: vi.fn(() => 42),
|
||||
}
|
||||
})
|
||||
|
||||
import { homedir } from 'node:os'
|
||||
|
||||
describe('handleDaemonCommands', () => {
|
||||
const appName = 'test-app'
|
||||
const mockTempDir = homedir()
|
||||
const pidFile = join(mockTempDir, '.shumai', 'pids', `${appName}.pid`)
|
||||
|
||||
let mockExit: MockInstance<typeof process.exit>
|
||||
let mockKill: MockInstance<typeof process.kill>
|
||||
const originalArgv = process.argv
|
||||
|
||||
beforeEach(() => {
|
||||
mkdirSync(mockTempDir, { recursive: true })
|
||||
mockExit = vi.spyOn(process, 'exit').mockImplementation(() => {
|
||||
throw new Error('process.exit')
|
||||
})
|
||||
mockKill = vi.spyOn(process, 'kill').mockImplementation(() => true)
|
||||
|
||||
vi.mocked(spawn).mockClear()
|
||||
vi.mocked(openSync).mockClear()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
process.argv = originalArgv
|
||||
vi.restoreAllMocks()
|
||||
try {
|
||||
rmSync(mockTempDir, { recursive: true, force: true })
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
})
|
||||
|
||||
it('runs the runFn on normal startup', async () => {
|
||||
process.argv = ['node', 'script.js']
|
||||
const runFn = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
await handleDaemonCommands(appName, runFn)
|
||||
|
||||
expect(runFn).toHaveBeenCalled()
|
||||
expect(existsSync(pidFile)).toBe(true)
|
||||
expect(readFileSync(pidFile, 'utf8').trim()).toBe(process.pid.toString())
|
||||
})
|
||||
|
||||
it('stops the process if stop command is passed', async () => {
|
||||
process.argv = ['node', 'script.js', 'stop']
|
||||
const runFn = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
// Write a dummy PID file
|
||||
mkdirSync(join(mockTempDir, '.shumai', 'pids'), { recursive: true })
|
||||
writeFileSync(pidFile, '12345')
|
||||
|
||||
await expect(handleDaemonCommands(appName, runFn)).rejects.toThrow('process.exit')
|
||||
|
||||
expect(runFn).not.toHaveBeenCalled()
|
||||
expect(mockKill).toHaveBeenCalledWith(12345, 'SIGTERM')
|
||||
expect(mockExit).toHaveBeenCalledWith(0)
|
||||
expect(openSync).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('daemonizes the process if -d is passed', async () => {
|
||||
process.argv = ['node', 'script.js', '-d']
|
||||
const runFn = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
await expect(handleDaemonCommands(appName, runFn)).rejects.toThrow('process.exit')
|
||||
|
||||
expect(runFn).not.toHaveBeenCalled()
|
||||
expect(spawn).toHaveBeenCalled()
|
||||
expect(existsSync(pidFile)).toBe(true)
|
||||
expect(readFileSync(pidFile, 'utf8').trim()).toBe('99999')
|
||||
expect(mockExit).toHaveBeenCalledWith(0)
|
||||
})
|
||||
|
||||
it('restarts the process if restart command is passed', async () => {
|
||||
process.argv = ['node', 'script.js', 'restart']
|
||||
const runFn = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
// Write a dummy PID file
|
||||
mkdirSync(join(mockTempDir, '.shumai', 'pids'), { recursive: true })
|
||||
writeFileSync(pidFile, '12345')
|
||||
|
||||
await expect(handleDaemonCommands(appName, runFn)).rejects.toThrow('process.exit')
|
||||
|
||||
expect(runFn).not.toHaveBeenCalled()
|
||||
expect(mockKill).toHaveBeenCalledWith(12345, 'SIGTERM')
|
||||
expect(spawn).toHaveBeenCalled()
|
||||
expect(mockExit).toHaveBeenCalledWith(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,269 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import {
|
||||
existsSync,
|
||||
writeFileSync,
|
||||
readFileSync,
|
||||
unlinkSync,
|
||||
mkdirSync,
|
||||
statSync,
|
||||
createReadStream,
|
||||
openSync,
|
||||
} from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { homedir } from 'node:os'
|
||||
|
||||
const SHUMAI_DIR = join(homedir(), '.shumai')
|
||||
const PID_DIR = join(SHUMAI_DIR, 'pids')
|
||||
const LOG_DIR = join(SHUMAI_DIR, 'logs')
|
||||
|
||||
function isProcessRunning(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return true
|
||||
} catch (err) {
|
||||
if (err && typeof err === 'object' && 'code' in err && err.code === 'EPERM') {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function stopProcess(appName: string, pidFile: string): Promise<boolean> {
|
||||
if (!existsSync(pidFile)) {
|
||||
console.log(`No running instance of ${appName} found.`)
|
||||
return false
|
||||
}
|
||||
|
||||
const pid = parseInt(readFileSync(pidFile, 'utf8').trim(), 10)
|
||||
if (isNaN(pid)) {
|
||||
console.log(`Invalid PID found in ${pidFile}. Cleaning up file.`)
|
||||
try {
|
||||
unlinkSync(pidFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
let isRunning = isProcessRunning(pid)
|
||||
if (!isRunning) {
|
||||
console.log(`Process ${pid} is not running. Cleaning up PID file.`)
|
||||
try {
|
||||
unlinkSync(pidFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
console.log(`Stopping ${appName} (PID: ${pid})...`)
|
||||
try {
|
||||
process.kill(pid, 'SIGTERM')
|
||||
} catch (err) {
|
||||
console.error(`Failed to send SIGTERM to process ${pid}:`, err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Wait for it to stop
|
||||
for (let i = 0; i < 30; i++) {
|
||||
if (!isProcessRunning(pid)) {
|
||||
isRunning = false
|
||||
break
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100))
|
||||
}
|
||||
|
||||
if (isRunning) {
|
||||
console.log(`Process ${pid} did not stop gracefully. Sending SIGKILL...`)
|
||||
try {
|
||||
process.kill(pid, 'SIGKILL')
|
||||
} catch (err) {
|
||||
console.error(`Failed to send SIGKILL to process ${pid}:`, err)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
unlinkSync(pidFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
|
||||
console.log(`Successfully stopped ${appName}.`)
|
||||
return true
|
||||
}
|
||||
|
||||
function startDaemon(appName: string, pidFile: string, logFile: string): void {
|
||||
// Check if already running
|
||||
if (existsSync(pidFile)) {
|
||||
const pid = parseInt(readFileSync(pidFile, 'utf8').trim(), 10)
|
||||
if (!isNaN(pid) && pid !== process.pid) {
|
||||
if (isProcessRunning(pid)) {
|
||||
console.error(
|
||||
`${appName} is already running (PID: ${pid}). Use '${appName} stop' or '${appName} restart'.`,
|
||||
)
|
||||
process.exit(1)
|
||||
} else {
|
||||
try {
|
||||
unlinkSync(pidFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Starting ${appName} in background...`)
|
||||
const logFd = openSync(logFile, 'w')
|
||||
|
||||
const childArgs = process.argv.slice(1).filter((arg) => arg !== '-d' && arg !== 'restart')
|
||||
|
||||
const child = spawn(process.argv[0], childArgs, {
|
||||
detached: true,
|
||||
stdio: ['ignore', logFd, logFd],
|
||||
env: {
|
||||
...process.env,
|
||||
SHUMAI_DAEMONIZED: 'true',
|
||||
},
|
||||
})
|
||||
|
||||
if (child.pid === undefined) {
|
||||
console.error(`Failed to spawn background process for ${appName}.`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
child.unref()
|
||||
writeFileSync(pidFile, child.pid.toString(), 'utf8')
|
||||
console.log(`🚀 Started ${appName} in background (PID: ${child.pid}).`)
|
||||
console.log(`📄 Logs are being written to ${logFile}`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
function viewLogs(appName: string, logFile: string): void {
|
||||
if (!existsSync(logFile)) {
|
||||
console.log(`No log file found for ${appName}.`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
// Print existing content (last 100 lines)
|
||||
const content = readFileSync(logFile, 'utf8')
|
||||
const lines = content.split('\n')
|
||||
const lastLines = lines.slice(-100).join('\n')
|
||||
process.stdout.write(lastLines + '\n')
|
||||
|
||||
let cursor = statSync(logFile).size
|
||||
const interval = setInterval(() => {
|
||||
try {
|
||||
if (existsSync(logFile)) {
|
||||
const stats = statSync(logFile)
|
||||
if (stats.size > cursor) {
|
||||
const stream = createReadStream(logFile, { start: cursor, end: stats.size - 1 })
|
||||
stream.pipe(process.stdout, { end: false })
|
||||
cursor = stats.size
|
||||
} else if (stats.size < cursor) {
|
||||
cursor = stats.size
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore read errors
|
||||
}
|
||||
}, 250)
|
||||
|
||||
const cleanup = () => {
|
||||
clearInterval(interval)
|
||||
process.exit(0)
|
||||
}
|
||||
process.on('SIGINT', cleanup)
|
||||
process.on('SIGTERM', cleanup)
|
||||
}
|
||||
|
||||
export async function handleDaemonCommands(
|
||||
appName: string,
|
||||
runFn: () => Promise<void>,
|
||||
): Promise<void> {
|
||||
const pidFile = join(PID_DIR, `${appName}.pid`)
|
||||
const logFile = join(LOG_DIR, `${appName}.log`)
|
||||
|
||||
// Ensure directories exist
|
||||
mkdirSync(PID_DIR, { recursive: true })
|
||||
mkdirSync(LOG_DIR, { recursive: true })
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
const isStop = args.includes('stop')
|
||||
const isRestart = args.includes('restart')
|
||||
const isLogs = args.includes('logs')
|
||||
const isDaemon = args.includes('-d') && process.env.SHUMAI_DAEMONIZED !== 'true'
|
||||
|
||||
if (isStop) {
|
||||
await stopProcess(appName, pidFile)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
if (isRestart) {
|
||||
await stopProcess(appName, pidFile)
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
startDaemon(appName, pidFile, logFile)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
if (isLogs) {
|
||||
viewLogs(appName, logFile)
|
||||
// Keep process open for logs tailing
|
||||
return new Promise(() => {})
|
||||
}
|
||||
|
||||
if (isDaemon) {
|
||||
startDaemon(appName, pidFile, logFile)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
// Normal Startup logic: check if already running
|
||||
if (existsSync(pidFile)) {
|
||||
const pid = parseInt(readFileSync(pidFile, 'utf8').trim(), 10)
|
||||
if (!isNaN(pid) && pid !== process.pid) {
|
||||
if (isProcessRunning(pid)) {
|
||||
console.error(
|
||||
`${appName} is already running (PID: ${pid}). Use '${appName} stop' or '${appName} restart'.`,
|
||||
)
|
||||
process.exit(1)
|
||||
} else {
|
||||
// Process is dead, clean up PID file
|
||||
try {
|
||||
unlinkSync(pidFile)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write the PID file
|
||||
writeFileSync(pidFile, process.pid.toString(), 'utf8')
|
||||
|
||||
const cleanPidFile = () => {
|
||||
try {
|
||||
if (existsSync(pidFile)) {
|
||||
const pidStr = readFileSync(pidFile, 'utf8').trim()
|
||||
if (parseInt(pidStr, 10) === process.pid) {
|
||||
unlinkSync(pidFile)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
// Register cleanups
|
||||
process.on('exit', cleanPidFile)
|
||||
|
||||
const handleSignal = (signal: string) => {
|
||||
if (process.listenerCount(signal) === 1) {
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => handleSignal('SIGINT'))
|
||||
process.on('SIGTERM', () => handleSignal('SIGTERM'))
|
||||
|
||||
// Run the main app
|
||||
await runFn()
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/db",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -5,20 +5,39 @@ import {
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuLabel,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
DropdownMenuSub,
|
||||
DropdownMenuSubTrigger,
|
||||
DropdownMenuSubContent,
|
||||
DropdownMenuPortal,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuSub,
|
||||
DropdownMenuSubContent,
|
||||
DropdownMenuSubTrigger,
|
||||
DropdownMenuTrigger,
|
||||
} from '@/ui/components/ui/dropdown-menu'
|
||||
import { useTeamId } from '@/ui/hooks/use-team-id'
|
||||
import { signOut } from '@/ui/lib/auth-client'
|
||||
import { useAuthStore } from '@/ui/stores/auth'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { useNavigate } from '@tanstack/react-router'
|
||||
import { useTheme } from './theme-provider'
|
||||
import { Check } from 'lucide-react'
|
||||
import { APP_VERSION } from '../version'
|
||||
import { useTheme } from './theme-provider'
|
||||
|
||||
const Github = (props: React.SVGProps<SVGSVGElement>) => (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="24"
|
||||
height="24"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
{...props}
|
||||
>
|
||||
<path d="M15 22v-4a4.8 4.8 0 0 0-1-3.5c3 0 6-2 6-5.5.08-1.25-.27-2.48-1-3.5.28-1.15.28-2.35 0-3.5 0 0-1 0-3 1.5-2.64-.5-5.36-.5-8 0C6 2 5 2 5 2c-.3 1.15-.3 2.35 0 3.5A5.403 5.403 0 0 0 4 9c0 3.5 3 5.5 6 5.5-.39.49-.68 1.05-.85 1.65-.17.6-.22 1.23-.15 1.85v4" />
|
||||
<path d="M9 18c-4.51 2-5-2-7-2" />
|
||||
</svg>
|
||||
)
|
||||
|
||||
export function UserMenu() {
|
||||
const { clearAuth } = useAuthStore()
|
||||
@@ -125,6 +144,16 @@ export function UserMenu() {
|
||||
<DropdownMenuItem className="text-red-500 focus:text-red-500" onClick={handleLogout}>
|
||||
Log out
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem
|
||||
className="text-xs text-muted-foreground flex items-center gap-2 cursor-pointer focus:bg-accent focus:text-accent-foreground justify-center bg-accent border"
|
||||
onClick={() =>
|
||||
window.open('https://github.com/shumaiOne/shumai', '_blank', 'noopener,noreferrer')
|
||||
}
|
||||
>
|
||||
<Github className="size-3.5" />
|
||||
<span>Version {APP_VERSION}</span>
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/webui",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./index.html",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export const APP_VERSION = '0.0.10'
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/workflow-core",
|
||||
"version": "0.0.8",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
+47
-12
@@ -1,5 +1,5 @@
|
||||
import tailwindPlugin from 'bun-plugin-tailwind'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { existsSync, readFileSync, readdirSync } from 'node:fs'
|
||||
import { cp, mkdir, readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { temporalWorkflow } from '../packages/workflow-core/src/bun-temporal-plugin'
|
||||
@@ -34,20 +34,48 @@ const commonExternal = [
|
||||
'@temporalio/worker',
|
||||
'@temporalio/workflow',
|
||||
'prisma',
|
||||
'zod',
|
||||
]
|
||||
|
||||
// Extract exact dependency versions from workspace
|
||||
// Extract exact dependency versions from workspace package.json files
|
||||
console.log('🔍 Extracting exact dependency versions for runtime packages...')
|
||||
const lsOutput = Bun.spawnSync(['bun', 'pm', 'ls', '--all']).stdout.toString()
|
||||
const dependencyVersions: Record<string, string> = {}
|
||||
|
||||
// Find all workspace package.json files
|
||||
const packageJsonPaths = ['package.json']
|
||||
const workspaceDirs = ['packages', 'apps']
|
||||
for (const dir of workspaceDirs) {
|
||||
if (existsSync(dir)) {
|
||||
const subdirs = readdirSync(dir)
|
||||
for (const subdir of subdirs) {
|
||||
const p = path.join(dir, subdir, 'package.json')
|
||||
if (existsSync(p)) {
|
||||
packageJsonPaths.push(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const dep of commonExternal) {
|
||||
const escapedDep = dep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
const regex = new RegExp(`(?:^|\\s)${escapedDep}@(\\d+\\.\\d+\\.\\d+[^\\s]*)`, 'g')
|
||||
const match = regex.exec(lsOutput)
|
||||
if (match && match[1]) {
|
||||
dependencyVersions[dep] = match[1]
|
||||
console.log(` ✅ Found ${dep}@${match[1]}`)
|
||||
let foundVersion: string | null = null
|
||||
for (const p of packageJsonPaths) {
|
||||
try {
|
||||
const content = readFileSync(p, 'utf8')
|
||||
const pkgJson = JSON.parse(content)
|
||||
const version = pkgJson.dependencies?.[dep] || pkgJson.devDependencies?.[dep]
|
||||
if (version) {
|
||||
// Strip any leading range specifiers like ^ or ~ to get the base version
|
||||
foundVersion = version.replace(/^[\^~]/, '')
|
||||
break
|
||||
}
|
||||
} catch {
|
||||
// ignore read/parse errors
|
||||
}
|
||||
}
|
||||
|
||||
if (foundVersion) {
|
||||
dependencyVersions[dep] = foundVersion
|
||||
console.log(` ✅ Found ${dep}@${foundVersion}`)
|
||||
} else {
|
||||
console.warn(` ⚠️ Warning: Could not find exact version for ${dep}`)
|
||||
}
|
||||
@@ -137,6 +165,7 @@ async function buildPlatformPackages(
|
||||
entrypoint: string,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
plugins: any[] = [],
|
||||
external?: string[],
|
||||
) {
|
||||
for (const target of targets) {
|
||||
const dirName = `${appName}-${target.platform}-${target.arch}`
|
||||
@@ -152,6 +181,7 @@ async function buildPlatformPackages(
|
||||
binaryName,
|
||||
bunTarget: target.bunTarget,
|
||||
plugins,
|
||||
external,
|
||||
})
|
||||
|
||||
// Generate package.json for platform-specific package
|
||||
@@ -181,6 +211,7 @@ async function buildMainPackage({
|
||||
entrypoint,
|
||||
plugins = [],
|
||||
extraDependencies = {},
|
||||
external = commonExternal,
|
||||
}: {
|
||||
appName: string
|
||||
description: string
|
||||
@@ -189,6 +220,7 @@ async function buildMainPackage({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
plugins?: any[]
|
||||
extraDependencies?: Record<string, string>
|
||||
external?: string[]
|
||||
}) {
|
||||
console.log(`\n📄 Packaging wrapper app ${appName}...`)
|
||||
const mainOutDir = path.join(distDir, appName)
|
||||
@@ -207,7 +239,7 @@ async function buildMainPackage({
|
||||
minify: true,
|
||||
naming: `${appName}-app.[ext]`,
|
||||
plugins,
|
||||
external: commonExternal,
|
||||
external,
|
||||
define: {
|
||||
'process.env.NODE_ENV': JSON.stringify('production'),
|
||||
},
|
||||
@@ -345,7 +377,8 @@ console.log(\`ℹ️ [${appName}] Running from CWD: \${process.cwd()}\`)
|
||||
console.log(\`ℹ️ [${appName}] DATABASE_URL is: \${process.env.DATABASE_URL ? '(defined)' : '(undefined)'}\`)
|
||||
|
||||
const prismaSchemaPath = join(__dirname, '..', 'prisma', 'schema.prisma')
|
||||
if (existsSync(prismaSchemaPath)) {
|
||||
const isUtilityCommand = process.argv.includes('stop') || process.argv.includes('logs')
|
||||
if (existsSync(prismaSchemaPath) && !isUtilityCommand) {
|
||||
console.log('🔄 Running database migrations...')
|
||||
activeProcess = spawn(
|
||||
binaryPath,
|
||||
@@ -460,13 +493,15 @@ const shumaiPlugins = [
|
||||
}),
|
||||
tailwindPlugin,
|
||||
]
|
||||
await buildPlatformPackages('shumai', dummyRunnerPath)
|
||||
const shumaiExternal = commonExternal.filter((dep) => dep !== 'zod')
|
||||
await buildPlatformPackages('shumai', dummyRunnerPath, shumaiPlugins, shumaiExternal)
|
||||
await buildMainPackage({
|
||||
appName: 'shumai',
|
||||
description: 'A fullstack AI-powered media workspace and workflow engine',
|
||||
hasPrisma: true,
|
||||
entrypoint: './apps/web/src/index.ts',
|
||||
plugins: shumaiPlugins,
|
||||
external: shumaiExternal,
|
||||
})
|
||||
|
||||
// 3. Build shumai-agent (Agent app & platform binaries)
|
||||
|
||||
@@ -91,6 +91,12 @@ function updateVersions(newVersion: string) {
|
||||
writeFileSync(file, JSON.stringify(content, null, 2) + '\n', 'utf-8')
|
||||
console.log(` Updated version in ${file}`)
|
||||
}
|
||||
|
||||
const versionFile = 'packages/webui/version.ts'
|
||||
if (existsSync(versionFile)) {
|
||||
writeFileSync(versionFile, `export const APP_VERSION = '${newVersion}'\n`, 'utf-8')
|
||||
console.log(` Updated version in ${versionFile}`)
|
||||
}
|
||||
}
|
||||
|
||||
function shellQuote(value: string) {
|
||||
|
||||
Reference in New Issue
Block a user