refactor(monorepo): decouple and split background workflow workers into packages

Splits monolithic workflow workers into four decoupled workspaces under the Bun workspaces monorepo:
- @shumai/workflow-core containing the core execution engine and utilities.
- @shumai/agent containing agent-specific logic, workflows, and activities.
- @shumai/transcode containing transcoding workflows, transcoder, and transcode activities.
- @shumai/worker-db containing database activities.

Introduces a dynamic workflow and activity registration pattern on LocalExecutor to eliminate circular dependencies. Updates worker bootstrapping and test suites to cleanly leverage generic workspace modules.
This commit is contained in:
yiling
2026-06-01 23:29:11 +08:00
parent cb80822637
commit e3f4fa219e
66 changed files with 421 additions and 150 deletions
+65
View File
@@ -145,6 +145,23 @@
"vitest": "^4.1.4",
},
},
"packages/agent": {
"name": "@shumai/agent",
"version": "0.1.0",
"dependencies": {
"@anthropic-ai/sandbox-runtime": "^0.0.52",
"@earendil-works/pi-agent-core": "0.78.0",
"@earendil-works/pi-ai": "0.75.4",
"@google/genai": "^1.50.1",
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@temporalio/activity": "^1.16.1",
"@temporalio/worker": "^1.16.1",
"@temporalio/workflow": "^1.16.1",
},
},
"packages/core": {
"name": "@shumai/core",
"version": "0.1.0",
@@ -178,6 +195,44 @@
"zod": "^4.4.3",
},
},
"packages/transcode": {
"name": "@shumai/transcode",
"version": "0.1.0",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@temporalio/activity": "^1.16.1",
"@temporalio/worker": "^1.16.1",
"@temporalio/workflow": "^1.16.1",
"sharp": "^0.34.5",
},
},
"packages/worker-db": {
"name": "@shumai/worker-db",
"version": "0.1.0",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@temporalio/activity": "^1.16.1",
"@temporalio/worker": "^1.16.1",
"@temporalio/workflow": "^1.16.1",
},
},
"packages/workflow-core": {
"name": "@shumai/workflow-core",
"version": "0.1.0",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@temporalio/client": "^1.16.1",
"@temporalio/workflow": "^1.16.1",
},
},
},
"packages": {
"@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="],
@@ -958,12 +1013,20 @@
"@shikijs/vscode-textmate": ["@shikijs/vscode-textmate@10.0.2", "", {}, "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg=="],
"@shumai/agent": ["@shumai/agent@workspace:packages/agent"],
"@shumai/core": ["@shumai/core@workspace:packages/core"],
"@shumai/db": ["@shumai/db@workspace:packages/db"],
"@shumai/dtos": ["@shumai/dtos@workspace:packages/dtos"],
"@shumai/transcode": ["@shumai/transcode@workspace:packages/transcode"],
"@shumai/worker-db": ["@shumai/worker-db@workspace:packages/worker-db"],
"@shumai/workflow-core": ["@shumai/workflow-core@workspace:packages/workflow-core"],
"@sinclair/typebox": ["@sinclair/typebox@0.34.49", "", {}, "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A=="],
"@smithy/chunked-blob-reader": ["@smithy/chunked-blob-reader@5.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw=="],
@@ -3288,6 +3351,8 @@
"@reduxjs/toolkit/immer": ["immer@11.1.8", "", {}, "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA=="],
"@shumai/agent/@google/genai": ["@google/genai@1.52.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q=="],
"@stoplight/better-ajv-errors/ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
"@stoplight/json/safe-stable-stringify": ["safe-stable-stringify@1.1.1", "", {}, "sha512-ERq4hUjKDbJfE4+XtZLFPCDi8Vb1JqaxAPTxWFLBx8XcAlf9Bda/ZJdVezs/NAfsMQScyIlUMx+Yeu7P7rx5jw=="],
+21
View File
@@ -0,0 +1,21 @@
{
"name": "@shumai/agent",
"version": "0.1.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
"dependencies": {
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@google/genai": "^1.50.1",
"@anthropic-ai/sandbox-runtime": "^0.0.52",
"@earendil-works/pi-agent-core": "0.78.0",
"@earendil-works/pi-ai": "0.75.4",
"@temporalio/activity": "^1.16.1",
"@temporalio/worker": "^1.16.1",
"@temporalio/workflow": "^1.16.1"
}
}
@@ -1,11 +1,11 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { agentChatActivity, autofillAiActivity, type AgentExecutionContext } from './agent'
import * as piAgent from '@/agent'
import * as piAgent from '../index'
import { type AgentHarness, type Session } from '@earendil-works/pi-agent-core'
import { type DatabaseSessionMetadata } from '@/agent/database-session-storage'
import { type DatabaseSessionMetadata } from '../database-session-storage'
vi.mock('@/agent', async () => {
const actual = await vi.importActual('@/agent')
vi.mock('../index', async () => {
const actual = await vi.importActual('../index')
return {
...actual,
createAgentSession: vi.fn(),
@@ -1,5 +1,5 @@
import { createAgentSession, fieldsToTypeBoxSchema, type AutofillField } from '@/agent'
import { DatabaseSessionStorage } from '@/agent/database-session-storage'
import { createAgentSession, fieldsToTypeBoxSchema, type AutofillField } from '../index'
import { DatabaseSessionStorage } from '../database-session-storage'
import type { Prisma, Skill } from '@shumai/db'
import { logger } from '@/logger'
import { s3Service } from '@shumai/core/src/s3/s3'
@@ -1,6 +1,6 @@
import { GoogleGenAI } from '@google/genai'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@/transcode/transcode'
import { transcodeService } from '@shumai/transcode'
import { exec } from 'child_process'
import * as fs from 'fs'
import * as os from 'os'
@@ -285,3 +285,41 @@ export function formatSkillsForPrompt(
return lines.join('\n')
}
import { registerWorkflow, registerActivities } from '@shumai/workflow-core'
import { WorkflowTaskType } from '@shumai/db'
import { agentEmbeddingMedia } from './workflows/agent-embedding'
import { queryEmbeddingForSearch } from './workflows/query-embedding-for-search'
import { agentAutofillMedia } from './workflows/agent-autofill'
import { agentChat } from './workflows/agent-chat'
import { agentToolCall } from './workflows/agent-tool-call'
import * as agentActivities from './activities/agent'
import * as aiActivities from './activities/ai'
export function initAgentWorkflows() {
registerWorkflow(WorkflowTaskType.ai_embedding, agentEmbeddingMedia)
registerWorkflow(WorkflowTaskType.query_embedding_for_search, queryEmbeddingForSearch)
registerWorkflow(WorkflowTaskType.ai_metadata_autofill, agentAutofillMedia)
registerWorkflow(WorkflowTaskType.chat, agentChat)
registerWorkflow(WorkflowTaskType.agent_tool_call, agentToolCall)
registerActivities(agentActivities)
registerActivities(aiActivities)
}
export * from './database-session-storage'
export * from './activities/agent'
export {
type GeneratedEmbedding,
type GenerateEmbeddingParams,
generateEmbeddingActivity,
type GenerateTextEmbeddingParams,
generateTextEmbeddingActivity,
type ExtractAiMetadataParams,
extractAiMetadataActivity,
} from './activities/ai'
export * from './workflows/agent-autofill'
export * from './workflows/agent-chat'
export * from './workflows/agent-embedding'
export * from './workflows/agent-tool-call'
export * from './workflows/query-embedding-for-search'
@@ -1,5 +1,5 @@
import { prisma } from '@shumai/db'
import { workflowService } from '@/workflow/workflow'
import { workflowService } from '@shumai/workflow-core'
import { WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
export async function executeAgentToolWorkflow(params: {
@@ -2,10 +2,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'
import { agentAutofillMedia } from './agent-autofill'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db'
import * as workflowUtils from '@/workflow/workflow-utils'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@/workflow/workflow-utils', async () => {
const actual = await vi.importActual('@/workflow/workflow-utils')
vi.mock('@shumai/workflow-core', async () => {
const actual = await vi.importActual('@shumai/workflow-core')
return {
...actual,
getActivities: vi.fn(),
@@ -6,7 +6,7 @@ import {
TaskQueueDb,
TaskQueueAgent,
TaskQueueTranscode,
} from '@/workflow/workflow-utils'
} from '@shumai/workflow-core'
export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
const {
@@ -28,7 +28,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
let placeholderCommentId: string | undefined
let tmpDir: string | undefined
let transcodeWorkerQueue: string | undefined
let transcodeWorkerQueue = ''
try {
// Update status to processing
@@ -120,11 +120,13 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
const aiResult = await executeActivity(agentWorkerQueue, autofillAiActivity, {
teamId,
images: generatedFiles,
fields: fields.map((f) => ({
id: f.key,
config: f.config as unknown as PrismaJson.FieldConfig,
description: f.description,
})),
fields: fields.map(
(f: { key: string; config: Record<string, unknown>; description?: string | null }) => ({
id: f.key,
config: f.config as unknown as PrismaJson.FieldConfig,
description: f.description,
}),
),
context,
})
@@ -2,10 +2,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'
import { agentChat } from './agent-chat'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db'
import * as workflowUtils from '@/workflow/workflow-utils'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@/workflow/workflow-utils', async () => {
const actual = await vi.importActual('@/workflow/workflow-utils')
vi.mock('@shumai/workflow-core', async () => {
const actual = await vi.importActual('@shumai/workflow-core')
return {
...actual,
getActivities: vi.fn(),
@@ -1,11 +1,6 @@
import { ApplicationFailure } from '@temporalio/workflow'
import type { WorkflowTask } from '@shumai/db'
import {
executeActivity,
getActivities,
TaskQueueAgent,
TaskQueueDb,
} from '@/workflow/workflow-utils'
import { executeActivity, getActivities, TaskQueueAgent, TaskQueueDb } from '@shumai/workflow-core'
export async function agentChat(task: WorkflowTask): Promise<void> {
const {
@@ -2,10 +2,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'
import { agentEmbeddingMedia } from './agent-embedding'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db'
import * as workflowUtils from '@/workflow/workflow-utils'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@/workflow/workflow-utils', async () => {
const actual = await vi.importActual('@/workflow/workflow-utils')
vi.mock('@shumai/workflow-core', async () => {
const actual = await vi.importActual('@shumai/workflow-core')
return {
...actual,
getActivities: vi.fn(),
@@ -1,6 +1,6 @@
import { ApplicationFailure } from '@temporalio/workflow'
import type { WorkflowTask } from '@shumai/db'
import { getActivities, executeActivity, TaskQueueDb, TaskQueueAgent } from '../workflow-utils'
import { getActivities, executeActivity, TaskQueueDb, TaskQueueAgent } from '@shumai/workflow-core'
export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
const {
@@ -1,6 +1,6 @@
import { ApplicationFailure } from '@temporalio/workflow'
import type { WorkflowTask } from '@shumai/db'
import { getActivities, executeActivity, TaskQueueDb } from '../workflow-utils'
import { getActivities, executeActivity, TaskQueueDb } from '@shumai/workflow-core'
export async function agentToolCall(task: WorkflowTask): Promise<void> {
const { updateTaskStatusActivity, executeAgentToolActivity } = getActivities()
@@ -1,5 +1,5 @@
import type { WorkflowTask } from '@shumai/db'
import { getActivities, executeActivity, TaskQueueDb } from '../workflow-utils'
import { getActivities, executeActivity, TaskQueueDb } from '@shumai/workflow-core'
export async function queryEmbeddingForSearch(task: WorkflowTask): Promise<void> {
const { generateTextEmbeddingActivity, updateWorkflowTaskActivity } = getActivities()
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"ignoreDeprecations": "6.0"
},
"include": ["src/**/*"]
}
+1 -1
View File
@@ -4,7 +4,7 @@ import { setupTestDbHooks } from '@shumai/db'
import { SearchService } from './search'
import { SearchConditionOperator } from '@shumai/dtos'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { workflowService } from '@/workflow/workflow'
import { workflowService } from '@shumai/workflow-core'
import { AssetType } from '@shumai/db'
+1 -1
View File
@@ -5,7 +5,7 @@ import { AssetInfo } from '@shumai/dtos'
import { SearchRequest } from '@shumai/dtos'
import { PaginatedData, decodeCursor, encodeCursor, PageInfo } from '@shumai/core/src/pagination'
import { generateSearchNgrams } from '@/utils/ngram'
import { workflowService } from '@/workflow/workflow'
import { workflowService } from '@shumai/workflow-core'
import { HTTPException } from 'hono/http-exception'
import { SqlQueryBuilder } from './sql-query-builder'
+1 -1
View File
@@ -20,7 +20,7 @@ import {
WorkflowTaskType,
} from '@shumai/db'
import { PaginationParams, paginateQuery, PaginatedData } from '@shumai/core/src/pagination'
import { VideoTranscoder, ImageTranscoder } from '@/transcode/transcoder'
import { VideoTranscoder, ImageTranscoder } from '@shumai/transcode'
export class UploadService {
constructor(private readonly prismaClient: typeof prisma = prisma) {}
+18
View File
@@ -0,0 +1,18 @@
{
"name": "@shumai/transcode",
"version": "0.1.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
"dependencies": {
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@temporalio/activity": "^1.16.1",
"@temporalio/worker": "^1.16.1",
"@temporalio/workflow": "^1.16.1",
"sharp": "^0.34.5"
}
}
@@ -3,7 +3,7 @@ import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db'
import { getMediaInfoActivity, transcodeVideoActivity, updateAssetMediaActivity } from './transcode'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@/transcode/transcode'
import { transcodeService } from '../transcode'
vi.mock('@shumai/core/src/s3/s3', () => ({
s3Service: {
@@ -14,7 +14,7 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
},
}))
vi.mock('@/transcode/transcode', () => ({
vi.mock('../transcode', () => ({
transcodeService: {
getVideoInfo: vi.fn(),
getImageInfo: vi.fn(),
@@ -1,6 +1,6 @@
import { prisma } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@/transcode/transcode'
import { transcodeService } from '../transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { ApplicationFailure } from '@temporalio/activity'
import * as path from 'path'
+14
View File
@@ -0,0 +1,14 @@
import { registerWorkflow, registerActivities } from '@shumai/workflow-core'
import { WorkflowTaskType } from '@shumai/db'
import { transcodeMedia } from './workflows/transcode'
import * as transcodeActivities from './activities/transcode'
export function initTranscodeWorkflows() {
registerWorkflow(WorkflowTaskType.transcode, transcodeMedia)
registerActivities(transcodeActivities)
}
export * from './transcoder'
export * from './transcode'
export * from './workflows/transcode'
export * from './activities/transcode'
@@ -1,14 +1,9 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { transcodeMedia } from './transcode'
import {
WorkflowTask,
WorkflowTaskStatus,
WorkflowTaskType,
AssetStatus,
} from '@shumai/db'
import * as workflowUtils from '@/workflow/workflow-utils'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@/workflow/workflow-utils', async (importOriginal) => {
vi.mock('@shumai/workflow-core', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = (await importOriginal()) as any
return {
@@ -5,7 +5,7 @@ import {
executeActivity,
TaskQueueDb,
TaskQueueTranscode,
} from '@/workflow/workflow-utils'
} from '@shumai/workflow-core'
export async function transcodeMedia(task: WorkflowTask): Promise<void> {
const {
@@ -23,7 +23,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
} = getActivities()
let tmpDir: string | undefined
let workerQueue: string | undefined
let workerQueue = ''
try {
// 1. Update status to processing
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"ignoreDeprecations": "6.0"
},
"include": ["src/**/*"]
}
+17
View File
@@ -0,0 +1,17 @@
{
"name": "@shumai/worker-db",
"version": "0.1.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
"dependencies": {
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@temporalio/activity": "^1.16.1",
"@temporalio/worker": "^1.16.1",
"@temporalio/workflow": "^1.16.1"
}
}
@@ -636,9 +636,10 @@ describe('Database Activities', () => {
})
const updatedFile3 = await prisma.asset.findUnique({ where: { id: file3Result.id } })
const freshFile2 = await prisma.asset.findUnique({ where: { id: file2Result.id } })
// Newest version (file3) must have a strictly lower sortIndex than version 2 (file2)
expect(updatedFile3!.sortIndex! < updatedFile2!.sortIndex!).toBe(true)
expect(updatedFile3!.sortIndex! < freshFile2!.sortIndex!).toBe(true)
})
})
})
@@ -5,7 +5,7 @@ import { ulid } from 'ulid'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { UpdateAssetMetadataRequest } from '@shumai/dtos'
import { WorkflowTaskStatus, AssetStatus, Prisma, AssetType } from '@shumai/db'
import type { AgentExecutionContext } from './agent'
import type { AgentExecutionContext } from '@shumai/agent'
import { assetService } from '@shumai/core/src/asset/asset'
import { VersionStackService } from '@shumai/core/src/versionStack/versionStack'
import { uploadService } from '@shumai/core/src/upload/upload'
+8
View File
@@ -0,0 +1,8 @@
import { registerActivities } from '@shumai/workflow-core'
import * as dbActivities from './activities/db'
export function initDbWorkflows() {
registerActivities(dbActivities)
}
export * from './activities/db'
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"ignoreDeprecations": "6.0"
},
"include": ["src/**/*"]
}
+15
View File
@@ -0,0 +1,15 @@
{
"name": "@shumai/workflow-core",
"version": "0.1.0",
"private": true,
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
"dependencies": {
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/core": "workspace:*",
"@temporalio/client": "^1.16.1",
"@temporalio/workflow": "^1.16.1"
}
}
@@ -1,7 +1,11 @@
import { describe, it, expect } from 'vitest'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db'
import { updateTaskStatusActivity, updateAssetStatusActivity, updateTaskUsageActivity } from './db'
import {
updateTaskStatusActivity,
updateAssetStatusActivity,
updateTaskUsageActivity,
} from '@shumai/worker-db'
import { WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db'
describe('Task Activities', () => {
+6
View File
@@ -0,0 +1,6 @@
export * from './executor'
export * from './local-executor'
export * from './temporal-executor'
export * from './workflow-utils'
export * from './workflow'
export * from './activities/task'
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db'
import { workflowService } from './workflow'
import { LocalExecutor, ConcurrencyLimiter } from './local-executor'
import { LocalExecutor, ConcurrencyLimiter, registerWorkflow } from './local-executor'
import { WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
// Mock the individual workflows
@@ -14,22 +14,6 @@ const mocks = {
transcodeMedia: vi.fn(),
}
vi.mock('./workflows/agent-embedding', () => ({
agentEmbeddingMedia: (task: unknown) => mocks.agentEmbeddingMedia(task),
}))
vi.mock('./workflows/query-embedding-for-search', () => ({
queryEmbeddingForSearch: (task: unknown) => mocks.queryEmbeddingForSearch(task),
}))
vi.mock('./workflows/agent-autofill', () => ({
agentAutofillMedia: (task: unknown) => mocks.agentAutofillMedia(task),
}))
vi.mock('./workflows/agent-chat', () => ({
agentChat: (task: unknown) => mocks.agentChat(task),
}))
vi.mock('@/transcode/workflows/transcode', () => ({
transcodeMedia: (task: unknown) => mocks.transcodeMedia(task),
}))
describe('LocalExecutor Integration Tests', () => {
setupTestDbHooks()
@@ -39,6 +23,17 @@ describe('LocalExecutor Integration Tests', () => {
// Get the global executor instance which processes tasks submitted via the Prisma Client Extension
executor = (workflowService as unknown as { executor: LocalExecutor }).executor
vi.clearAllMocks()
// Register workflows dynamically for testing
registerWorkflow(WorkflowTaskType.ai_embedding, async (task) => mocks.agentEmbeddingMedia(task))
registerWorkflow(WorkflowTaskType.query_embedding_for_search, async (task) =>
mocks.queryEmbeddingForSearch(task),
)
registerWorkflow(WorkflowTaskType.ai_metadata_autofill, async (task) =>
mocks.agentAutofillMedia(task),
)
registerWorkflow(WorkflowTaskType.chat, async (task) => mocks.agentChat(task))
registerWorkflow(WorkflowTaskType.transcode, async (task) => mocks.transcodeMedia(task))
})
describe('Heartbeat & Task Status transition', () => {
@@ -1,20 +1,33 @@
import { prisma } from '@shumai/db'
import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import { Executor } from './executor'
import { agentEmbeddingMedia } from './workflows/agent-embedding'
import { queryEmbeddingForSearch } from './workflows/query-embedding-for-search'
import { agentAutofillMedia } from './workflows/agent-autofill'
import { agentChat } from './workflows/agent-chat'
import { transcodeMedia } from '@/transcode/workflows/transcode'
import { agentToolCall } from './workflows/agent-tool-call'
import { activities } from './activities/index'
import * as taskActivities from './activities/task'
import { logger } from '@/logger'
type WorkflowFn = (task: WorkflowTask) => Promise<void>
const workflowRegistry = new Map<string, WorkflowFn>()
const activityRegistry: Record<string, unknown> = { ...taskActivities }
export function registerWorkflow(type: string, fn: WorkflowFn) {
workflowRegistry.set(type, fn)
}
/* eslint-disable @typescript-eslint/naming-convention */
const globalObj = globalThis as typeof globalThis & {
__localActivities?: Record<string, unknown>
__localLogger?: unknown
}
/* eslint-enable @typescript-eslint/naming-convention */
export function registerActivities(acts: Record<string, unknown>) {
Object.assign(activityRegistry, acts)
globalObj.__localActivities = activityRegistry
}
// Inject local dependencies into global scope for workflow-utils
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(globalThis as any).__localActivities = activities
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(globalThis as any).__localLogger = logger
globalObj.__localActivities = activityRegistry
globalObj.__localLogger = logger
export class ConcurrencyLimiter {
private activeCount = 0
@@ -235,27 +248,11 @@ export class LocalExecutor implements Executor {
private async processTaskWrapper(task: WorkflowTask) {
try {
switch (task.type) {
case WorkflowTaskType.ai_embedding:
await agentEmbeddingMedia(task)
break
case WorkflowTaskType.query_embedding_for_search:
await queryEmbeddingForSearch(task)
break
case WorkflowTaskType.ai_metadata_autofill:
await agentAutofillMedia(task)
break
case WorkflowTaskType.chat:
await agentChat(task)
break
case WorkflowTaskType.transcode:
await transcodeMedia(task)
break
case WorkflowTaskType.agent_tool_call:
await agentToolCall(task)
break
default:
console.warn(`Unknown task type: ${task.type}`)
const fn = task.type ? workflowRegistry.get(task.type) : undefined
if (fn) {
await fn(task)
} else {
console.warn(`Unknown or unregistered task type: ${task.type}`)
}
} catch (err) {
console.error(`Error in workflow ${task.type} for task ${task.id}:`, err)
@@ -1,5 +1,4 @@
// Use type-only imports for activities to avoid bundling them in Temporal workflows
import type { activities } from './activities/index'
import * as wf from '@temporalio/workflow'
/**
@@ -22,12 +21,25 @@ export function isTemporal(): boolean {
}
}
interface LocalLogger {
debug(obj: Record<string, unknown>, msg: string): void
info(obj: Record<string, unknown>, msg: string): void
error(obj: Record<string, unknown>, msg: string): void
warn(obj: Record<string, unknown>, msg: string): void
}
/* eslint-disable @typescript-eslint/naming-convention */
const globalObj = globalThis as typeof globalThis & {
__localActivities?: Record<string, unknown>
__localLogger?: LocalLogger
}
/* eslint-enable @typescript-eslint/naming-convention */
/**
* A proxy that mimics proxyActivities for local execution.
*/
function localActivitiesProxy(): typeof activities {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const acts = (globalThis as any).__localActivities
function localActivitiesProxy(): Record<string, unknown> {
const acts = globalObj.__localActivities
if (!acts) {
throw new Error('Local activities not injected into globalThis.__localActivities')
}
@@ -48,11 +60,17 @@ export async function sleep(ms: number): Promise<void> {
/**
* Environment-aware activity proxy.
*/
export function getActivities(): typeof activities {
// getActivities must default to `any` so that workflows can dynamically
// destructure and call any activity when they do not supply a precise type parameter.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function getActivities<T = any>(): T {
// baseProxy is dynamically proxying all activities, so we must type it as any.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let baseProxy: any
if (isTemporal()) {
baseProxy = wf.proxyActivities<typeof activities>({
// Temporal's proxyActivities requires a type parameter, so we use any for generic activities.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
baseProxy = wf.proxyActivities<any>({
startToCloseTimeout: '10 minutes',
})
} else {
@@ -93,8 +111,7 @@ function getLogger() {
warn: (obj: Record<string, unknown>, msg: string) => wf.log.warn(msg, obj),
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return (globalThis as any).__localLogger || console
return globalObj.__localLogger || console
}
/**
@@ -103,11 +120,29 @@ function getLogger() {
* @param activityFunc The proxy function obtained from `getActivities()`
* @param args The arguments to pass to the activity
*/
// Overload signature for untyped or generic activity callbacks.
export async function executeActivity(
queue: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
activityFunc: any,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...args: any[]
// eslint-disable-next-line @typescript-eslint/no-explicit-any
): Promise<any>
export async function executeActivity<TArgs extends unknown[], TRet>(
queue: string,
activityFunc: (...args: TArgs) => Promise<TRet>,
...args: TArgs
): Promise<TRet> {
): Promise<TRet>
// Implementation signature for executeActivity.
export async function executeActivity(
queue: string,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
activityFunc: any,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...args: any[]
// eslint-disable-next-line @typescript-eslint/no-explicit-any
): Promise<any> {
// We extract the hidden _activityName property attached by getActivities()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const activityName = (activityFunc as any)._activityName
@@ -121,7 +156,9 @@ export async function executeActivity<TArgs extends unknown[], TRet>(
logger.debug({ queue, activityName }, `Executing activity ${activityName} on queue ${queue}`)
try {
let result: TRet
// result holds the untyped return value from the dynamic activity proxy, so we type it as any.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let result: any
if (isTemporal()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const handle = wf.proxyActivities<any>({
@@ -2,6 +2,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { prisma } from '@shumai/db'
import { workflowService } from './workflow'
import { WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
import { initAgentWorkflows } from '@shumai/agent'
import { initTranscodeWorkflows } from '@shumai/transcode'
import { initDbWorkflows } from '@shumai/worker-db'
vi.mock('@shumai/core/src/s3/s3', () => ({
s3Service: {
@@ -13,22 +16,29 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
},
}))
vi.mock('@/transcode/transcode', () => ({
transcodeService: {
createTempDir: vi.fn().mockReturnValue('/tmp/shumai-test'),
removeDir: vi.fn(),
extractAudio: vi.fn(),
extractVideoFrames: vi.fn().mockResolvedValue([]),
getVideoInfo: vi.fn(),
getImageInfo: vi.fn(),
transcodeVideo: vi.fn(),
transcodeImage: vi.fn(),
generateSprite: vi.fn(),
},
}))
vi.mock('@shumai/transcode', async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>
return {
...actual,
transcodeService: {
createTempDir: vi.fn().mockReturnValue('/tmp/shumai-test'),
removeDir: vi.fn(),
extractAudio: vi.fn(),
extractVideoFrames: vi.fn().mockResolvedValue([]),
getVideoInfo: vi.fn(),
getImageInfo: vi.fn(),
transcodeVideo: vi.fn(),
transcodeImage: vi.fn(),
generateSprite: vi.fn(),
},
}
})
describe('WorkflowService', () => {
beforeEach(async () => {
initAgentWorkflows()
initTranscodeWorkflows()
initDbWorkflows()
await prisma.workflowTask.deleteMany()
await prisma.assetComment.deleteMany()
await prisma.asset.deleteMany()
@@ -56,18 +56,26 @@ export class WorkflowService {
}
// Starts Temporal workers for specific task queues
async startWorkers(queue: string = 'db_queue'): Promise<void> {
async startWorkers(
queue: string = 'db_queue',
options?: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
workflowBundle?: any
// eslint-disable-next-line @typescript-eslint/no-explicit-any
activities?: any
},
): Promise<void> {
const type = process.env.WORKFLOW_EXECUTOR || 'local'
if (type === 'temporal') {
const { Worker, NativeConnection } = await import('@temporalio/worker')
const { activities } = await import('./activities/index')
const connection = await NativeConnection.connect({
address: process.env.TEMPORAL_ADDRESS || 'localhost:7233',
})
// @ts-ignore
const { default: workflowBundle } = await import('./workflows:::workflow')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const activities = options?.activities || (globalThis as any).__localActivities
const workflowBundle = options?.workflowBundle
if (queue === 'db_queue') {
console.log(`Starting Temporal worker for queue: ${queue}`)
+7
View File
@@ -0,0 +1,7 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"ignoreDeprecations": "6.0"
},
"include": ["src/**/*"]
}
+9 -1
View File
@@ -24,7 +24,10 @@ import uploadRoute from '@/api/upload'
import versionStackRoute from '@/api/versionStack'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { assetService } from '@shumai/core/src/asset/asset'
import { workflowService } from '@/workflow/workflow'
import { workflowService } from '@shumai/workflow-core'
import { initAgentWorkflows } from '@shumai/agent'
import { initTranscodeWorkflows } from '@shumai/transcode'
import { initDbWorkflows } from '@shumai/worker-db'
type User = Prisma.UserGetPayload<Record<string, never>>
@@ -62,6 +65,11 @@ const apiAppRoute = app.route('/api', apiRoute)
export type AppType = typeof apiAppRoute
export async function run() {
// Initialize workflows and activities for local executor mode
initAgentWorkflows()
initTranscodeWorkflows()
initDbWorkflows()
// Start services
await metadataService.syncSystemFields().catch(console.error)
assetService.startCleanupJob()
+10 -2
View File
@@ -1,7 +1,15 @@
import { workflowService } from '@/workflow/workflow'
import { TaskQueueAgent, TaskQueueTranscode } from '@/workflow/workflow-utils'
import { workflowService } from '@shumai/workflow-core'
import { TaskQueueAgent, TaskQueueTranscode } from '@shumai/workflow-core'
import { initAgentWorkflows } from '@shumai/agent'
import { initTranscodeWorkflows } from '@shumai/transcode'
import { initDbWorkflows } from '@shumai/worker-db'
export async function run() {
// Initialize workflows and activities
initAgentWorkflows()
initTranscodeWorkflows()
initDbWorkflows()
const domain = process.argv[3]
if (!domain) {
-13
View File
@@ -1,13 +0,0 @@
import * as transcodeActivities from '@/transcode/activities/transcode'
import * as agentActivities from './agent'
import * as aiActivities from './ai'
import * as dbActivities from './db'
import * as taskActivities from './task'
export const activities = {
...taskActivities,
...aiActivities,
...transcodeActivities,
...agentActivities,
...dbActivities,
}
-6
View File
@@ -1,6 +0,0 @@
export * from './agent-embedding'
export * from './agent-autofill'
export * from './agent-chat'
export * from './query-embedding-for-search'
export * from './agent-tool-call'
export * from '@/transcode/workflows/transcode'