mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 11:07:18 +00:00
feat(workflow): allow configuring transcode and agent concurrency via environment variables
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { workflowService } from './workflow'
|
||||
@@ -345,4 +345,46 @@ describe('LocalExecutor Integration Tests', () => {
|
||||
expect(limiter.getQueueLength()).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Environment Variable Concurrency Config', () => {
|
||||
let originalTranscode: string | undefined
|
||||
let originalAgent: string | undefined
|
||||
|
||||
beforeEach(() => {
|
||||
originalTranscode = process.env.CONCURRENCY_TRANSCODE
|
||||
originalAgent = process.env.CONCURRENCY_AGENT
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (originalTranscode === undefined) {
|
||||
delete process.env.CONCURRENCY_TRANSCODE
|
||||
} else {
|
||||
process.env.CONCURRENCY_TRANSCODE = originalTranscode
|
||||
}
|
||||
|
||||
if (originalAgent === undefined) {
|
||||
delete process.env.CONCURRENCY_AGENT
|
||||
} else {
|
||||
process.env.CONCURRENCY_AGENT = originalAgent
|
||||
}
|
||||
})
|
||||
|
||||
it('should initialize with values from environment variables', () => {
|
||||
process.env.CONCURRENCY_TRANSCODE = '2'
|
||||
process.env.CONCURRENCY_AGENT = '8'
|
||||
|
||||
const customExecutor = new LocalExecutor()
|
||||
expect(customExecutor.getTranscodeConcurrencyLimit()).toBe(2)
|
||||
expect(customExecutor.getAgentConcurrencyLimit()).toBe(8)
|
||||
})
|
||||
|
||||
it('should fallback to defaults when environment variables are not set', () => {
|
||||
delete process.env.CONCURRENCY_TRANSCODE
|
||||
delete process.env.CONCURRENCY_AGENT
|
||||
|
||||
const defaultExecutor = new LocalExecutor()
|
||||
expect(defaultExecutor.getTranscodeConcurrencyLimit()).toBe(1)
|
||||
expect(defaultExecutor.getAgentConcurrencyLimit()).toBe(5)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -35,6 +35,10 @@ export class ConcurrencyLimiter {
|
||||
|
||||
constructor(private readonly limit: number) {}
|
||||
|
||||
getLimit(): number {
|
||||
return this.limit
|
||||
}
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (this.activeCount >= this.limit) {
|
||||
await new Promise<void>((resolve) => this.queue.push(resolve))
|
||||
@@ -68,9 +72,28 @@ export class LocalExecutor implements Executor {
|
||||
private interval: Timer | null = null
|
||||
private processingTasks = new Set<string>()
|
||||
|
||||
// 1 concurrency for transcode, 5 for other workflows
|
||||
private transcodeLimiter = new ConcurrencyLimiter(1)
|
||||
private generalLimiter = new ConcurrencyLimiter(5)
|
||||
private transcodeLimiter: ConcurrencyLimiter
|
||||
private generalLimiter: ConcurrencyLimiter
|
||||
|
||||
constructor() {
|
||||
const transcodeLimit = process.env.CONCURRENCY_TRANSCODE
|
||||
? parseInt(process.env.CONCURRENCY_TRANSCODE, 10)
|
||||
: 1
|
||||
const agentLimit = process.env.CONCURRENCY_AGENT
|
||||
? parseInt(process.env.CONCURRENCY_AGENT, 10)
|
||||
: 5
|
||||
|
||||
this.transcodeLimiter = new ConcurrencyLimiter(transcodeLimit)
|
||||
this.generalLimiter = new ConcurrencyLimiter(agentLimit)
|
||||
}
|
||||
|
||||
getTranscodeConcurrencyLimit(): number {
|
||||
return this.transcodeLimiter.getLimit()
|
||||
}
|
||||
|
||||
getAgentConcurrencyLimit(): number {
|
||||
return this.generalLimiter.getLimit()
|
||||
}
|
||||
|
||||
async submit(task: WorkflowTask): Promise<string> {
|
||||
if (task.status !== WorkflowTaskStatus.pending) {
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { workflowService } from './workflow'
|
||||
import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils'
|
||||
|
||||
const mockCreate = vi.fn().mockResolvedValue({
|
||||
run: vi.fn().mockResolvedValue(undefined),
|
||||
})
|
||||
|
||||
const mockConnect = vi.fn().mockResolvedValue({})
|
||||
|
||||
vi.mock('@temporalio/worker', () => ({
|
||||
Worker: {
|
||||
create: (options: Record<string, unknown>) => mockCreate(options),
|
||||
},
|
||||
NativeConnection: {
|
||||
connect: (options: Record<string, unknown>) => mockConnect(options),
|
||||
},
|
||||
}))
|
||||
|
||||
describe('WorkflowService Temporal Workers Concurrency Control', () => {
|
||||
let originalExecutor: string | undefined
|
||||
let originalTranscode: string | undefined
|
||||
let originalAgent: string | undefined
|
||||
|
||||
beforeEach(() => {
|
||||
originalExecutor = process.env.WORKFLOW_EXECUTOR
|
||||
originalTranscode = process.env.CONCURRENCY_TRANSCODE
|
||||
originalAgent = process.env.CONCURRENCY_AGENT
|
||||
process.env.WORKFLOW_EXECUTOR = 'temporal'
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (originalExecutor === undefined) {
|
||||
delete process.env.WORKFLOW_EXECUTOR
|
||||
} else {
|
||||
process.env.WORKFLOW_EXECUTOR = originalExecutor
|
||||
}
|
||||
|
||||
if (originalTranscode === undefined) {
|
||||
delete process.env.CONCURRENCY_TRANSCODE
|
||||
} else {
|
||||
process.env.CONCURRENCY_TRANSCODE = originalTranscode
|
||||
}
|
||||
|
||||
if (originalAgent === undefined) {
|
||||
delete process.env.CONCURRENCY_AGENT
|
||||
} else {
|
||||
process.env.CONCURRENCY_AGENT = originalAgent
|
||||
}
|
||||
})
|
||||
|
||||
it('should pass default concurrency limits to specific workers', async () => {
|
||||
delete process.env.CONCURRENCY_TRANSCODE
|
||||
delete process.env.CONCURRENCY_AGENT
|
||||
|
||||
// Test Transcode Worker
|
||||
await workflowService.startWorkers(TaskQueueTranscode)
|
||||
// Worker.create should be called twice (shared worker + specific worker)
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
|
||||
// The second call is for the specific worker
|
||||
const transcodeSpecificWorkerOptions = mockCreate.mock.calls[1][0]
|
||||
expect(transcodeSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(1)
|
||||
|
||||
vi.clearAllMocks()
|
||||
|
||||
// Test Agent Worker
|
||||
await workflowService.startWorkers(TaskQueueAgent)
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
|
||||
const agentSpecificWorkerOptions = mockCreate.mock.calls[1][0]
|
||||
expect(agentSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(5)
|
||||
})
|
||||
|
||||
it('should pass environment variable overridden concurrency limits to specific workers', async () => {
|
||||
process.env.CONCURRENCY_TRANSCODE = '3'
|
||||
process.env.CONCURRENCY_AGENT = '12'
|
||||
|
||||
// Test Transcode Worker
|
||||
await workflowService.startWorkers(TaskQueueTranscode)
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
|
||||
const transcodeSpecificWorkerOptions = mockCreate.mock.calls[1][0]
|
||||
expect(transcodeSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(3)
|
||||
|
||||
vi.clearAllMocks()
|
||||
|
||||
// Test Agent Worker
|
||||
await workflowService.startWorkers(TaskQueueAgent)
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
|
||||
const agentSpecificWorkerOptions = mockCreate.mock.calls[1][0]
|
||||
expect(agentSpecificWorkerOptions.maxConcurrentActivityTaskExecutions).toBe(12)
|
||||
})
|
||||
})
|
||||
@@ -5,6 +5,7 @@ import * as taskActivities from './activities/task'
|
||||
import { Executor } from './executor'
|
||||
import { LocalExecutor } from './local-executor'
|
||||
import { TemporalExecutor } from './temporal-executor'
|
||||
import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils'
|
||||
|
||||
export class WorkflowService {
|
||||
private executor: Executor
|
||||
@@ -117,11 +118,23 @@ export class WorkflowService {
|
||||
},
|
||||
})
|
||||
|
||||
let maxConcurrentActivityTaskExecutions: number | undefined
|
||||
if (queue === TaskQueueTranscode) {
|
||||
maxConcurrentActivityTaskExecutions = process.env.CONCURRENCY_TRANSCODE
|
||||
? parseInt(process.env.CONCURRENCY_TRANSCODE, 10)
|
||||
: 1
|
||||
} else if (queue === TaskQueueAgent) {
|
||||
maxConcurrentActivityTaskExecutions = process.env.CONCURRENCY_AGENT
|
||||
? parseInt(process.env.CONCURRENCY_AGENT, 10)
|
||||
: 5
|
||||
}
|
||||
|
||||
const specificWorker = await Worker.create({
|
||||
connection,
|
||||
// Register both domain-specific activities and shared task activities
|
||||
activities: { ...activities, ...taskActivities },
|
||||
taskQueue: workerSpecificQueue,
|
||||
maxConcurrentActivityTaskExecutions,
|
||||
bundlerOptions: {
|
||||
webpackConfigHook: (config) => {
|
||||
config.resolve = config.resolve || {}
|
||||
|
||||
Reference in New Issue
Block a user