From b8d876d28e9d3a8aaa3d93ff7141897a45ce4fe2 Mon Sep 17 00:00:00 2001 From: yiling Date: Tue, 16 Jun 2026 10:24:45 +0800 Subject: [PATCH] feat(workflow): allow configuring transcode and agent concurrency via environment variables --- .../workflow-core/src/local-executor.test.ts | 44 ++++++++- packages/workflow-core/src/local-executor.ts | 29 +++++- .../src/workflow-workers.test.ts | 96 +++++++++++++++++++ packages/workflow-core/src/workflow.ts | 13 +++ 4 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 packages/workflow-core/src/workflow-workers.test.ts diff --git a/packages/workflow-core/src/local-executor.test.ts b/packages/workflow-core/src/local-executor.test.ts index 26385b44..040a3a29 100644 --- a/packages/workflow-core/src/local-executor.test.ts +++ b/packages/workflow-core/src/local-executor.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db/test' import { workflowService } from './workflow' @@ -345,4 +345,46 @@ describe('LocalExecutor Integration Tests', () => { expect(limiter.getQueueLength()).toBe(0) }) }) + + describe('Environment Variable Concurrency Config', () => { + let originalTranscode: string | undefined + let originalAgent: string | undefined + + beforeEach(() => { + originalTranscode = process.env.CONCURRENCY_TRANSCODE + originalAgent = process.env.CONCURRENCY_AGENT + }) + + afterEach(() => { + if (originalTranscode === undefined) { + delete process.env.CONCURRENCY_TRANSCODE + } else { + process.env.CONCURRENCY_TRANSCODE = originalTranscode + } + + if (originalAgent === undefined) { + delete process.env.CONCURRENCY_AGENT + } else { + process.env.CONCURRENCY_AGENT = originalAgent + } + }) + + it('should initialize with values from environment variables', () => { + process.env.CONCURRENCY_TRANSCODE = '2' + process.env.CONCURRENCY_AGENT = '8' + + const customExecutor = new LocalExecutor() + expect(customExecutor.getTranscodeConcurrencyLimit()).toBe(2) + expect(customExecutor.getAgentConcurrencyLimit()).toBe(8) + }) + + it('should fallback to defaults when environment variables are not set', () => { + delete process.env.CONCURRENCY_TRANSCODE + delete process.env.CONCURRENCY_AGENT + + const defaultExecutor = new LocalExecutor() + expect(defaultExecutor.getTranscodeConcurrencyLimit()).toBe(1) + expect(defaultExecutor.getAgentConcurrencyLimit()).toBe(5) + }) + }) }) diff --git a/packages/workflow-core/src/local-executor.ts b/packages/workflow-core/src/local-executor.ts index cf5d5f92..273221bd 100644 --- a/packages/workflow-core/src/local-executor.ts +++ b/packages/workflow-core/src/local-executor.ts @@ -35,6 +35,10 @@ export class ConcurrencyLimiter { constructor(private readonly limit: number) {} + getLimit(): number { + return this.limit + } + async run(fn: () => Promise): Promise { if (this.activeCount >= this.limit) { await new Promise((resolve) => this.queue.push(resolve)) @@ -68,9 +72,28 @@ export class LocalExecutor implements Executor { private interval: Timer | null = null private processingTasks = new Set() - // 1 concurrency for transcode, 5 for other workflows - private transcodeLimiter = new ConcurrencyLimiter(1) - private generalLimiter = new ConcurrencyLimiter(5) + private transcodeLimiter: ConcurrencyLimiter + private generalLimiter: ConcurrencyLimiter + + constructor() { + const transcodeLimit = process.env.CONCURRENCY_TRANSCODE + ? parseInt(process.env.CONCURRENCY_TRANSCODE, 10) + : 1 + const agentLimit = process.env.CONCURRENCY_AGENT + ? parseInt(process.env.CONCURRENCY_AGENT, 10) + : 5 + + this.transcodeLimiter = new ConcurrencyLimiter(transcodeLimit) + this.generalLimiter = new ConcurrencyLimiter(agentLimit) + } + + getTranscodeConcurrencyLimit(): number { + return this.transcodeLimiter.getLimit() + } + + getAgentConcurrencyLimit(): number { + return this.generalLimiter.getLimit() + } async submit(task: WorkflowTask): Promise { if (task.status !== WorkflowTaskStatus.pending) { diff --git a/packages/workflow-core/src/workflow-workers.test.ts b/packages/workflow-core/src/workflow-workers.test.ts new file mode 100644 index 00000000..3b3e20d7 --- /dev/null +++ b/packages/workflow-core/src/workflow-workers.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { workflowService } from './workflow' +import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils' + +const mockCreate = vi.fn().mockResolvedValue({ + run: vi.fn().mockResolvedValue(undefined), +}) + +const mockConnect = vi.fn().mockResolvedValue({}) + +vi.mock('@temporalio/worker', () => ({ + Worker: { + create: (options: Record) => mockCreate(options), + }, + NativeConnection: { + connect: (options: Record) => mockConnect(options), + }, +})) + +describe('WorkflowService Temporal Workers Concurrency Control', () => { + let originalExecutor: string | undefined + let originalTranscode: string | undefined + let originalAgent: string | undefined + + beforeEach(() => { + originalExecutor = process.env.WORKFLOW_EXECUTOR + originalTranscode = process.env.CONCURRENCY_TRANSCODE + originalAgent = process.env.CONCURRENCY_AGENT + process.env.WORKFLOW_EXECUTOR = 'temporal' + vi.clearAllMocks() + }) + + afterEach(() => { + if (originalExecutor === undefined) { + delete process.env.WORKFLOW_EXECUTOR + } else { + process.env.WORKFLOW_EXECUTOR = originalExecutor + } + + if (originalTranscode === undefined) { + delete process.env.CONCURRENCY_TRANSCODE + } else { + process.env.CONCURRENCY_TRANSCODE = originalTranscode + } + + if (originalAgent === undefined) { + delete process.env.CONCURRENCY_AGENT + } else { + process.env.CONCURRENCY_AGENT = originalAgent + } + }) + + it('should pass default concurrency limits to specific workers', async () => { + delete process.env.CONCURRENCY_TRANSCODE + delete process.env.CONCURRENCY_AGENT + + // Test Transcode Worker + await workflowService.startWorkers(TaskQueueTranscode) + // Worker.create should be called twice (shared worker + specific worker) + expect(mockCreate).toHaveBeenCalledTimes(2) + + // The second call is for the specific worker + const transcodeSpecificWorkerOptions = mockCreate.mock.calls[1][0] + expect(transcodeSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(1) + + vi.clearAllMocks() + + // Test Agent Worker + await workflowService.startWorkers(TaskQueueAgent) + expect(mockCreate).toHaveBeenCalledTimes(2) + + const agentSpecificWorkerOptions = mockCreate.mock.calls[1][0] + expect(agentSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(5) + }) + + it('should pass environment variable overridden concurrency limits to specific workers', async () => { + process.env.CONCURRENCY_TRANSCODE = '3' + process.env.CONCURRENCY_AGENT = '12' + + // Test Transcode Worker + await workflowService.startWorkers(TaskQueueTranscode) + expect(mockCreate).toHaveBeenCalledTimes(2) + + const transcodeSpecificWorkerOptions = mockCreate.mock.calls[1][0] + expect(transcodeSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(3) + + vi.clearAllMocks() + + // Test Agent Worker + await workflowService.startWorkers(TaskQueueAgent) + expect(mockCreate).toHaveBeenCalledTimes(2) + + const agentSpecificWorkerOptions = mockCreate.mock.calls[1][0] + expect(agentSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(12) + }) +}) diff --git a/packages/workflow-core/src/workflow.ts b/packages/workflow-core/src/workflow.ts index f437d72e..9f8eb048 100644 --- a/packages/workflow-core/src/workflow.ts +++ b/packages/workflow-core/src/workflow.ts @@ -5,6 +5,7 @@ import * as taskActivities from './activities/task' import { Executor } from './executor' import { LocalExecutor } from './local-executor' import { TemporalExecutor } from './temporal-executor' +import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils' export class WorkflowService { private executor: Executor @@ -117,11 +118,23 @@ export class WorkflowService { }, }) + let maxConcurrentActivityTaskExecutions: number | undefined + if (queue === TaskQueueTranscode) { + maxConcurrentActivityTaskExecutions = process.env.CONCURRENCY_TRANSCODE + ? parseInt(process.env.CONCURRENCY_TRANSCODE, 10) + : 1 + } else if (queue === TaskQueueAgent) { + maxConcurrentActivityTaskExecutions = process.env.CONCURRENCY_AGENT + ? parseInt(process.env.CONCURRENCY_AGENT, 10) + : 5 + } + const specificWorker = await Worker.create({ connection, // Register both domain-specific activities and shared task activities activities: { ...activities, ...taskActivities }, taskQueue: workerSpecificQueue, + maxConcurrentActivityTaskExecutions, bundlerOptions: { webpackConfigHook: (config) => { config.resolve = config.resolve || {}