From 0d5fe48a3ac95a2c654552485b23daa6982cf636 Mon Sep 17 00:00:00 2001 From: Yiling-J Date: Wed, 21 Oct 2026 12:45:09 +0800 Subject: [PATCH] docs(docker): add guide for scaling transcoding workers horizontally (#486) --- README.md | 32 +++++++++++++++ packages/core/src/metadata/metadata.ts | 54 ++++++++++++++++---------- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 105e572b..142139ca 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,38 @@ Docker Compose is the fastest way to get Shumai running. You do not need to clon ``` 6. Open your browser and access Shumai at `http://localhost:3000` (or `http://:3000` for remote deployments). +#### **Scaling Transcode Workers (Parallel Processing)** + +By default, the Docker Compose deployment transcodes videos one-by-one. This is by design, as video transcoding is compute-intensive and can saturate available CPU or GPU resources. + +However, when uploading large batches (e.g., 100+ videos at once) or massive raw camera files that take hours to process and block subsequent uploads, you can scale transcoding throughput across additional machines. Each machine still processes tasks sequentially, but adding more worker instances increases overall parallel capacity. + +Shumai makes this seamless with a dedicated [`shumaione/shumai-transcode`](https://hub.docker.com/r/shumaione/shumai-transcode) Docker image: + +1. **Connect**: Launch a temporary machine and ensure network access to your PostgreSQL database and storage backend (e.g., S3 or shared storage). +2. **Start the worker**: Run the container with matching database and storage credentials: + + ```bash + docker run -d --name shumai_transcode_worker \ + -e DATABASE_URL="postgres://shumai:shumai_password@:5432/shumai_db?sslmode=disable" \ + -e STORAGE_BACKEND="s3" \ + -e S3_BUCKET="your-bucket-name" \ + -e AWS_ENDPOINT_URL_S3="https://s3.us-east-1.amazonaws.com" \ + -e S3_ACCESS_KEY_ID="your-access-key-id" \ + -e S3_SECRET_ACCESS_KEY="your-secret-access-key" \ + shumaione/shumai-transcode:latest + ``` + + The worker will automatically attach to the task queue and begin processing pending transcode jobs in parallel with your primary instance. + +3. **Stop anytime**: Once the transcoding backlog finishes (or whenever you no longer need the extra compute), simply stop or remove the worker: + + ```bash + docker stop shumai_transcode_worker + ``` + + There is no downtime or impact on the main Shumai application. + ### Option 2: Run from Source (Development) To set up Shumai locally for development: diff --git a/packages/core/src/metadata/metadata.ts b/packages/core/src/metadata/metadata.ts index 507ccebf..681f399a 100644 --- a/packages/core/src/metadata/metadata.ts +++ b/packages/core/src/metadata/metadata.ts @@ -43,24 +43,30 @@ export function getNextOptionColor(existingOptions: PrismaJson.SelectOption[] = export class MetadataService { constructor(private client: typeof prisma = prisma) {} async syncSystemFields(): Promise { - await this.client.$transaction(async (tx) => { - for (const field of systemFields) { - await tx.metadataField.upsert({ - where: { key: field.key }, - update: { - scope: field.scope, - readOnly: field.readOnly, - config: field.config, - }, - create: { - key: field.key, - scope: field.scope, - readOnly: field.readOnly ?? false, - config: field.config, - }, - }) - } - }) + await this.client.$transaction( + async (tx) => { + for (const field of systemFields) { + await tx.metadataField.upsert({ + where: { key: field.key }, + update: { + scope: field.scope, + readOnly: field.readOnly, + config: field.config, + }, + create: { + key: field.key, + scope: field.scope, + readOnly: field.readOnly ?? false, + config: field.config, + }, + }) + } + }, + { + maxWait: 10000, + timeout: 30000, + }, + ) } async createTeamField( @@ -312,9 +318,15 @@ export class MetadataService { reqs: UpdateAssetMetadataRequest[], allowReadOnly = false, ): Promise { - await this.client.$transaction(async (tx) => { - await this.updateAssetMetadataInternal(tx, assetId, reqs, allowReadOnly) - }) + await this.client.$transaction( + async (tx) => { + await this.updateAssetMetadataInternal(tx, assetId, reqs, allowReadOnly) + }, + { + maxWait: 10000, + timeout: 30000, + }, + ) } /**