From e3f4fa219e49a2911522b42e02562962a7360b11 Mon Sep 17 00:00:00 2001 From: yiling Date: Mon, 1 Jun 2026 23:29:11 +0800 Subject: [PATCH] refactor(monorepo): decouple and split background workflow workers into packages Splits monolithic workflow workers into four decoupled workspaces under the Bun workspaces monorepo: - @shumai/workflow-core containing the core execution engine and utilities. - @shumai/agent containing agent-specific logic, workflows, and activities. - @shumai/transcode containing transcoding workflows, transcoder, and transcode activities. - @shumai/worker-db containing database activities. Introduces a dynamic workflow and activity registration pattern on LocalExecutor to eliminate circular dependencies. Updates worker bootstrapping and test suites to cleanly leverage generic workspace modules. --- bun.lock | 65 +++++++++++++++++++ packages/agent/package.json | 21 ++++++ .../agent/src}/activities/agent.test.ts | 8 +-- .../agent/src}/activities/agent.ts | 4 +- .../agent/src}/activities/ai.test.ts | 0 .../agent/src}/activities/ai.ts | 2 +- .../src}/database-session-storage.test.ts | 0 .../agent/src}/database-session-storage.ts | 0 .../agent/src}/index.test.ts | 0 {src/agent => packages/agent/src}/index.ts | 38 +++++++++++ .../agent/src}/schema-conversion.test.ts | 0 .../src}/tools/analyze-asset-media.test.ts | 0 .../agent/src}/tools/analyze-asset-media.ts | 0 .../agent/src}/tools/create-file.ts | 0 .../agent/src}/tools/create-folder.ts | 0 .../agent/src}/tools/create-version.ts | 0 .../agent/src}/tools/list-assets.ts | 0 .../agent/src}/tools/read-skill.test.ts | 0 .../agent/src}/tools/read-skill.ts | 0 .../agent/src}/tools/sandboxed-bash.test.ts | 0 .../agent/src}/tools/sandboxed-bash.ts | 0 .../agent/src}/tools/utils.ts | 2 +- .../src}/workflows/agent-autofill.test.ts | 6 +- .../agent/src}/workflows/agent-autofill.ts | 16 +++-- .../agent/src}/workflows/agent-chat.test.ts | 6 +- .../agent/src}/workflows/agent-chat.ts | 7 +- .../src}/workflows/agent-embedding.test.ts | 6 +- .../agent/src}/workflows/agent-embedding.ts | 2 +- .../agent/src}/workflows/agent-tool-call.ts | 2 +- .../workflows/query-embedding-for-search.ts | 2 +- packages/agent/tsconfig.json | 7 ++ packages/core/src/search/search.test.ts | 2 +- packages/core/src/search/search.ts | 2 +- packages/core/src/upload/upload.ts | 2 +- packages/transcode/package.json | 18 +++++ .../src}/activities/transcode.test.ts | 4 +- .../transcode/src}/activities/transcode.ts | 2 +- packages/transcode/src/index.ts | 14 ++++ .../transcode/src}/transcode.test.ts | 0 .../transcode/src}/transcode.ts | 0 .../transcode/src}/transcoder.ts | 0 .../src}/workflows/transcode.test.ts | 11 +--- .../transcode/src}/workflows/transcode.ts | 4 +- packages/transcode/tsconfig.json | 7 ++ packages/worker-db/package.json | 17 +++++ .../worker-db/src}/activities/db.test.ts | 3 +- .../worker-db/src}/activities/db.ts | 2 +- packages/worker-db/src/index.ts | 8 +++ packages/worker-db/tsconfig.json | 7 ++ packages/workflow-core/package.json | 15 +++++ .../src}/activities/task.test.ts | 6 +- .../workflow-core/src}/activities/task.ts | 0 .../workflow-core/src}/executor.ts | 0 packages/workflow-core/src/index.ts | 6 ++ .../workflow-core/src}/local-executor.test.ts | 29 ++++----- .../workflow-core/src}/local-executor.ts | 61 +++++++++-------- .../workflow-core/src}/temporal-executor.ts | 0 .../workflow-core/src}/workflow-utils.test.ts | 0 .../workflow-core/src}/workflow-utils.ts | 57 +++++++++++++--- .../workflow-core/src}/workflow.test.ts | 36 ++++++---- .../workflow-core/src}/workflow.ts | 16 +++-- packages/workflow-core/tsconfig.json | 7 ++ src/cli/serve.ts | 10 ++- src/cli/worker.ts | 12 +++- src/workflow/activities/index.ts | 13 ---- src/workflow/workflows/index.ts | 6 -- 66 files changed, 421 insertions(+), 150 deletions(-) create mode 100644 packages/agent/package.json rename {src/workflow => packages/agent/src}/activities/agent.test.ts (94%) rename {src/workflow => packages/agent/src}/activities/agent.ts (98%) rename {src/workflow => packages/agent/src}/activities/ai.test.ts (100%) rename {src/workflow => packages/agent/src}/activities/ai.ts (99%) rename {src/agent => packages/agent/src}/database-session-storage.test.ts (100%) rename {src/agent => packages/agent/src}/database-session-storage.ts (100%) rename {src/agent => packages/agent/src}/index.test.ts (100%) rename {src/agent => packages/agent/src}/index.ts (85%) rename {src/agent => packages/agent/src}/schema-conversion.test.ts (100%) rename {src/agent => packages/agent/src}/tools/analyze-asset-media.test.ts (100%) rename {src/agent => packages/agent/src}/tools/analyze-asset-media.ts (100%) rename {src/agent => packages/agent/src}/tools/create-file.ts (100%) rename {src/agent => packages/agent/src}/tools/create-folder.ts (100%) rename {src/agent => packages/agent/src}/tools/create-version.ts (100%) rename {src/agent => packages/agent/src}/tools/list-assets.ts (100%) rename {src/agent => packages/agent/src}/tools/read-skill.test.ts (100%) rename {src/agent => packages/agent/src}/tools/read-skill.ts (100%) rename {src/agent => packages/agent/src}/tools/sandboxed-bash.test.ts (100%) rename {src/agent => packages/agent/src}/tools/sandboxed-bash.ts (100%) rename {src/agent => packages/agent/src}/tools/utils.ts (94%) rename {src/workflow => packages/agent/src}/workflows/agent-autofill.test.ts (95%) rename {src/workflow => packages/agent/src}/workflows/agent-autofill.ts (94%) rename {src/workflow => packages/agent/src}/workflows/agent-chat.test.ts (95%) rename {src/workflow => packages/agent/src}/workflows/agent-chat.ts (98%) rename {src/workflow => packages/agent/src}/workflows/agent-embedding.test.ts (94%) rename {src/workflow => packages/agent/src}/workflows/agent-embedding.ts (98%) rename {src/workflow => packages/agent/src}/workflows/agent-tool-call.ts (94%) rename {src/workflow => packages/agent/src}/workflows/query-embedding-for-search.ts (94%) create mode 100644 packages/agent/tsconfig.json create mode 100644 packages/transcode/package.json rename {src/transcode => packages/transcode/src}/activities/transcode.test.ts (97%) rename {src/transcode => packages/transcode/src}/activities/transcode.ts (99%) create mode 100644 packages/transcode/src/index.ts rename {src/transcode => packages/transcode/src}/transcode.test.ts (100%) rename {src/transcode => packages/transcode/src}/transcode.ts (100%) rename {src/transcode => packages/transcode/src}/transcoder.ts (100%) rename {src/transcode => packages/transcode/src}/workflows/transcode.test.ts (96%) rename {src/transcode => packages/transcode/src}/workflows/transcode.ts (99%) create mode 100644 packages/transcode/tsconfig.json create mode 100644 packages/worker-db/package.json rename {src/workflow => packages/worker-db/src}/activities/db.test.ts (99%) rename {src/workflow => packages/worker-db/src}/activities/db.ts (99%) create mode 100644 packages/worker-db/src/index.ts create mode 100644 packages/worker-db/tsconfig.json create mode 100644 packages/workflow-core/package.json rename {src/workflow => packages/workflow-core/src}/activities/task.test.ts (94%) rename {src/workflow => packages/workflow-core/src}/activities/task.ts (100%) rename {src/workflow => packages/workflow-core/src}/executor.ts (100%) create mode 100644 packages/workflow-core/src/index.ts rename {src/workflow => packages/workflow-core/src}/local-executor.test.ts (94%) rename {src/workflow => packages/workflow-core/src}/local-executor.ts (84%) rename {src/workflow => packages/workflow-core/src}/temporal-executor.ts (100%) rename {src/workflow => packages/workflow-core/src}/workflow-utils.test.ts (100%) rename {src/workflow => packages/workflow-core/src}/workflow-utils.ts (68%) rename {src/workflow => packages/workflow-core/src}/workflow.test.ts (87%) rename {src/workflow => packages/workflow-core/src}/workflow.ts (88%) create mode 100644 packages/workflow-core/tsconfig.json delete mode 100644 src/workflow/activities/index.ts delete mode 100644 src/workflow/workflows/index.ts diff --git a/bun.lock b/bun.lock index b1bdd048..219abf77 100644 --- a/bun.lock +++ b/bun.lock @@ -145,6 +145,23 @@ "vitest": "^4.1.4", }, }, + "packages/agent": { + "name": "@shumai/agent", + "version": "0.1.0", + "dependencies": { + "@anthropic-ai/sandbox-runtime": "^0.0.52", + "@earendil-works/pi-agent-core": "0.78.0", + "@earendil-works/pi-ai": "0.75.4", + "@google/genai": "^1.50.1", + "@shumai/core": "workspace:*", + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "@temporalio/activity": "^1.16.1", + "@temporalio/worker": "^1.16.1", + "@temporalio/workflow": "^1.16.1", + }, + }, "packages/core": { "name": "@shumai/core", "version": "0.1.0", @@ -178,6 +195,44 @@ "zod": "^4.4.3", }, }, + "packages/transcode": { + "name": "@shumai/transcode", + "version": "0.1.0", + "dependencies": { + "@shumai/core": "workspace:*", + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "@temporalio/activity": "^1.16.1", + "@temporalio/worker": "^1.16.1", + "@temporalio/workflow": "^1.16.1", + "sharp": "^0.34.5", + }, + }, + "packages/worker-db": { + "name": "@shumai/worker-db", + "version": "0.1.0", + "dependencies": { + "@shumai/core": "workspace:*", + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "@temporalio/activity": "^1.16.1", + "@temporalio/worker": "^1.16.1", + "@temporalio/workflow": "^1.16.1", + }, + }, + "packages/workflow-core": { + "name": "@shumai/workflow-core", + "version": "0.1.0", + "dependencies": { + "@shumai/core": "workspace:*", + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@temporalio/client": "^1.16.1", + "@temporalio/workflow": "^1.16.1", + }, + }, }, "packages": { "@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="], @@ -958,12 +1013,20 @@ "@shikijs/vscode-textmate": ["@shikijs/vscode-textmate@10.0.2", "", {}, "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg=="], + "@shumai/agent": ["@shumai/agent@workspace:packages/agent"], + "@shumai/core": ["@shumai/core@workspace:packages/core"], "@shumai/db": ["@shumai/db@workspace:packages/db"], "@shumai/dtos": ["@shumai/dtos@workspace:packages/dtos"], + "@shumai/transcode": ["@shumai/transcode@workspace:packages/transcode"], + + "@shumai/worker-db": ["@shumai/worker-db@workspace:packages/worker-db"], + + "@shumai/workflow-core": ["@shumai/workflow-core@workspace:packages/workflow-core"], + "@sinclair/typebox": ["@sinclair/typebox@0.34.49", "", {}, "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A=="], "@smithy/chunked-blob-reader": ["@smithy/chunked-blob-reader@5.2.2", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw=="], @@ -3288,6 +3351,8 @@ "@reduxjs/toolkit/immer": ["immer@11.1.8", "", {}, "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA=="], + "@shumai/agent/@google/genai": ["@google/genai@1.52.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q=="], + "@stoplight/better-ajv-errors/ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], "@stoplight/json/safe-stable-stringify": ["safe-stable-stringify@1.1.1", "", {}, "sha512-ERq4hUjKDbJfE4+XtZLFPCDi8Vb1JqaxAPTxWFLBx8XcAlf9Bda/ZJdVezs/NAfsMQScyIlUMx+Yeu7P7rx5jw=="], diff --git a/packages/agent/package.json b/packages/agent/package.json new file mode 100644 index 00000000..9287924a --- /dev/null +++ b/packages/agent/package.json @@ -0,0 +1,21 @@ +{ + "name": "@shumai/agent", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/core": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "@google/genai": "^1.50.1", + "@anthropic-ai/sandbox-runtime": "^0.0.52", + "@earendil-works/pi-agent-core": "0.78.0", + "@earendil-works/pi-ai": "0.75.4", + "@temporalio/activity": "^1.16.1", + "@temporalio/worker": "^1.16.1", + "@temporalio/workflow": "^1.16.1" + } +} diff --git a/src/workflow/activities/agent.test.ts b/packages/agent/src/activities/agent.test.ts similarity index 94% rename from src/workflow/activities/agent.test.ts rename to packages/agent/src/activities/agent.test.ts index cec9088a..5c87dbdd 100644 --- a/src/workflow/activities/agent.test.ts +++ b/packages/agent/src/activities/agent.test.ts @@ -1,11 +1,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { agentChatActivity, autofillAiActivity, type AgentExecutionContext } from './agent' -import * as piAgent from '@/agent' +import * as piAgent from '../index' import { type AgentHarness, type Session } from '@earendil-works/pi-agent-core' -import { type DatabaseSessionMetadata } from '@/agent/database-session-storage' +import { type DatabaseSessionMetadata } from '../database-session-storage' -vi.mock('@/agent', async () => { - const actual = await vi.importActual('@/agent') +vi.mock('../index', async () => { + const actual = await vi.importActual('../index') return { ...actual, createAgentSession: vi.fn(), diff --git a/src/workflow/activities/agent.ts b/packages/agent/src/activities/agent.ts similarity index 98% rename from src/workflow/activities/agent.ts rename to packages/agent/src/activities/agent.ts index f465a5d0..f62e736a 100644 --- a/src/workflow/activities/agent.ts +++ b/packages/agent/src/activities/agent.ts @@ -1,5 +1,5 @@ -import { createAgentSession, fieldsToTypeBoxSchema, type AutofillField } from '@/agent' -import { DatabaseSessionStorage } from '@/agent/database-session-storage' +import { createAgentSession, fieldsToTypeBoxSchema, type AutofillField } from '../index' +import { DatabaseSessionStorage } from '../database-session-storage' import type { Prisma, Skill } from '@shumai/db' import { logger } from '@/logger' import { s3Service } from '@shumai/core/src/s3/s3' diff --git a/src/workflow/activities/ai.test.ts b/packages/agent/src/activities/ai.test.ts similarity index 100% rename from src/workflow/activities/ai.test.ts rename to packages/agent/src/activities/ai.test.ts diff --git a/src/workflow/activities/ai.ts b/packages/agent/src/activities/ai.ts similarity index 99% rename from src/workflow/activities/ai.ts rename to packages/agent/src/activities/ai.ts index a67049a0..19f422a3 100644 --- a/src/workflow/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -1,6 +1,6 @@ import { GoogleGenAI } from '@google/genai' import { s3Service } from '@shumai/core/src/s3/s3' -import { transcodeService } from '@/transcode/transcode' +import { transcodeService } from '@shumai/transcode' import { exec } from 'child_process' import * as fs from 'fs' import * as os from 'os' diff --git a/src/agent/database-session-storage.test.ts b/packages/agent/src/database-session-storage.test.ts similarity index 100% rename from src/agent/database-session-storage.test.ts rename to packages/agent/src/database-session-storage.test.ts diff --git a/src/agent/database-session-storage.ts b/packages/agent/src/database-session-storage.ts similarity index 100% rename from src/agent/database-session-storage.ts rename to packages/agent/src/database-session-storage.ts diff --git a/src/agent/index.test.ts b/packages/agent/src/index.test.ts similarity index 100% rename from src/agent/index.test.ts rename to packages/agent/src/index.test.ts diff --git a/src/agent/index.ts b/packages/agent/src/index.ts similarity index 85% rename from src/agent/index.ts rename to packages/agent/src/index.ts index fc007f5e..99fbeb29 100644 --- a/src/agent/index.ts +++ b/packages/agent/src/index.ts @@ -285,3 +285,41 @@ export function formatSkillsForPrompt( return lines.join('\n') } + +import { registerWorkflow, registerActivities } from '@shumai/workflow-core' +import { WorkflowTaskType } from '@shumai/db' +import { agentEmbeddingMedia } from './workflows/agent-embedding' +import { queryEmbeddingForSearch } from './workflows/query-embedding-for-search' +import { agentAutofillMedia } from './workflows/agent-autofill' +import { agentChat } from './workflows/agent-chat' +import { agentToolCall } from './workflows/agent-tool-call' +import * as agentActivities from './activities/agent' +import * as aiActivities from './activities/ai' + +export function initAgentWorkflows() { + registerWorkflow(WorkflowTaskType.ai_embedding, agentEmbeddingMedia) + registerWorkflow(WorkflowTaskType.query_embedding_for_search, queryEmbeddingForSearch) + registerWorkflow(WorkflowTaskType.ai_metadata_autofill, agentAutofillMedia) + registerWorkflow(WorkflowTaskType.chat, agentChat) + registerWorkflow(WorkflowTaskType.agent_tool_call, agentToolCall) + + registerActivities(agentActivities) + registerActivities(aiActivities) +} + +export * from './database-session-storage' +export * from './activities/agent' +export { + type GeneratedEmbedding, + type GenerateEmbeddingParams, + generateEmbeddingActivity, + type GenerateTextEmbeddingParams, + generateTextEmbeddingActivity, + type ExtractAiMetadataParams, + extractAiMetadataActivity, +} from './activities/ai' +export * from './workflows/agent-autofill' +export * from './workflows/agent-chat' +export * from './workflows/agent-embedding' +export * from './workflows/agent-tool-call' +export * from './workflows/query-embedding-for-search' diff --git a/src/agent/schema-conversion.test.ts b/packages/agent/src/schema-conversion.test.ts similarity index 100% rename from src/agent/schema-conversion.test.ts rename to packages/agent/src/schema-conversion.test.ts diff --git a/src/agent/tools/analyze-asset-media.test.ts b/packages/agent/src/tools/analyze-asset-media.test.ts similarity index 100% rename from src/agent/tools/analyze-asset-media.test.ts rename to packages/agent/src/tools/analyze-asset-media.test.ts diff --git a/src/agent/tools/analyze-asset-media.ts b/packages/agent/src/tools/analyze-asset-media.ts similarity index 100% rename from src/agent/tools/analyze-asset-media.ts rename to packages/agent/src/tools/analyze-asset-media.ts diff --git a/src/agent/tools/create-file.ts b/packages/agent/src/tools/create-file.ts similarity index 100% rename from src/agent/tools/create-file.ts rename to packages/agent/src/tools/create-file.ts diff --git a/src/agent/tools/create-folder.ts b/packages/agent/src/tools/create-folder.ts similarity index 100% rename from src/agent/tools/create-folder.ts rename to packages/agent/src/tools/create-folder.ts diff --git a/src/agent/tools/create-version.ts b/packages/agent/src/tools/create-version.ts similarity index 100% rename from src/agent/tools/create-version.ts rename to packages/agent/src/tools/create-version.ts diff --git a/src/agent/tools/list-assets.ts b/packages/agent/src/tools/list-assets.ts similarity index 100% rename from src/agent/tools/list-assets.ts rename to packages/agent/src/tools/list-assets.ts diff --git a/src/agent/tools/read-skill.test.ts b/packages/agent/src/tools/read-skill.test.ts similarity index 100% rename from src/agent/tools/read-skill.test.ts rename to packages/agent/src/tools/read-skill.test.ts diff --git a/src/agent/tools/read-skill.ts b/packages/agent/src/tools/read-skill.ts similarity index 100% rename from src/agent/tools/read-skill.ts rename to packages/agent/src/tools/read-skill.ts diff --git a/src/agent/tools/sandboxed-bash.test.ts b/packages/agent/src/tools/sandboxed-bash.test.ts similarity index 100% rename from src/agent/tools/sandboxed-bash.test.ts rename to packages/agent/src/tools/sandboxed-bash.test.ts diff --git a/src/agent/tools/sandboxed-bash.ts b/packages/agent/src/tools/sandboxed-bash.ts similarity index 100% rename from src/agent/tools/sandboxed-bash.ts rename to packages/agent/src/tools/sandboxed-bash.ts diff --git a/src/agent/tools/utils.ts b/packages/agent/src/tools/utils.ts similarity index 94% rename from src/agent/tools/utils.ts rename to packages/agent/src/tools/utils.ts index 5a7ef91a..1d4defb8 100644 --- a/src/agent/tools/utils.ts +++ b/packages/agent/src/tools/utils.ts @@ -1,5 +1,5 @@ import { prisma } from '@shumai/db' -import { workflowService } from '@/workflow/workflow' +import { workflowService } from '@shumai/workflow-core' import { WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db' export async function executeAgentToolWorkflow(params: { diff --git a/src/workflow/workflows/agent-autofill.test.ts b/packages/agent/src/workflows/agent-autofill.test.ts similarity index 95% rename from src/workflow/workflows/agent-autofill.test.ts rename to packages/agent/src/workflows/agent-autofill.test.ts index 28a14749..e4a83a3d 100644 --- a/src/workflow/workflows/agent-autofill.test.ts +++ b/packages/agent/src/workflows/agent-autofill.test.ts @@ -2,10 +2,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { agentAutofillMedia } from './agent-autofill' import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db' -import * as workflowUtils from '@/workflow/workflow-utils' +import * as workflowUtils from '@shumai/workflow-core' -vi.mock('@/workflow/workflow-utils', async () => { - const actual = await vi.importActual('@/workflow/workflow-utils') +vi.mock('@shumai/workflow-core', async () => { + const actual = await vi.importActual('@shumai/workflow-core') return { ...actual, getActivities: vi.fn(), diff --git a/src/workflow/workflows/agent-autofill.ts b/packages/agent/src/workflows/agent-autofill.ts similarity index 94% rename from src/workflow/workflows/agent-autofill.ts rename to packages/agent/src/workflows/agent-autofill.ts index 884a5ac1..f6de1e23 100644 --- a/src/workflow/workflows/agent-autofill.ts +++ b/packages/agent/src/workflows/agent-autofill.ts @@ -6,7 +6,7 @@ import { TaskQueueDb, TaskQueueAgent, TaskQueueTranscode, -} from '@/workflow/workflow-utils' +} from '@shumai/workflow-core' export async function agentAutofillMedia(task: WorkflowTask): Promise { const { @@ -28,7 +28,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { let placeholderCommentId: string | undefined let tmpDir: string | undefined - let transcodeWorkerQueue: string | undefined + let transcodeWorkerQueue = '' try { // Update status to processing @@ -120,11 +120,13 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { const aiResult = await executeActivity(agentWorkerQueue, autofillAiActivity, { teamId, images: generatedFiles, - fields: fields.map((f) => ({ - id: f.key, - config: f.config as unknown as PrismaJson.FieldConfig, - description: f.description, - })), + fields: fields.map( + (f: { key: string; config: Record; description?: string | null }) => ({ + id: f.key, + config: f.config as unknown as PrismaJson.FieldConfig, + description: f.description, + }), + ), context, }) diff --git a/src/workflow/workflows/agent-chat.test.ts b/packages/agent/src/workflows/agent-chat.test.ts similarity index 95% rename from src/workflow/workflows/agent-chat.test.ts rename to packages/agent/src/workflows/agent-chat.test.ts index 093fa30f..46854518 100644 --- a/src/workflow/workflows/agent-chat.test.ts +++ b/packages/agent/src/workflows/agent-chat.test.ts @@ -2,10 +2,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { agentChat } from './agent-chat' import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db' -import * as workflowUtils from '@/workflow/workflow-utils' +import * as workflowUtils from '@shumai/workflow-core' -vi.mock('@/workflow/workflow-utils', async () => { - const actual = await vi.importActual('@/workflow/workflow-utils') +vi.mock('@shumai/workflow-core', async () => { + const actual = await vi.importActual('@shumai/workflow-core') return { ...actual, getActivities: vi.fn(), diff --git a/src/workflow/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts similarity index 98% rename from src/workflow/workflows/agent-chat.ts rename to packages/agent/src/workflows/agent-chat.ts index 973f044d..7228d1e5 100644 --- a/src/workflow/workflows/agent-chat.ts +++ b/packages/agent/src/workflows/agent-chat.ts @@ -1,11 +1,6 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { - executeActivity, - getActivities, - TaskQueueAgent, - TaskQueueDb, -} from '@/workflow/workflow-utils' +import { executeActivity, getActivities, TaskQueueAgent, TaskQueueDb } from '@shumai/workflow-core' export async function agentChat(task: WorkflowTask): Promise { const { diff --git a/src/workflow/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts similarity index 94% rename from src/workflow/workflows/agent-embedding.test.ts rename to packages/agent/src/workflows/agent-embedding.test.ts index ab6a99f5..2301166f 100644 --- a/src/workflow/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -2,10 +2,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { agentEmbeddingMedia } from './agent-embedding' import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db' -import * as workflowUtils from '@/workflow/workflow-utils' +import * as workflowUtils from '@shumai/workflow-core' -vi.mock('@/workflow/workflow-utils', async () => { - const actual = await vi.importActual('@/workflow/workflow-utils') +vi.mock('@shumai/workflow-core', async () => { + const actual = await vi.importActual('@shumai/workflow-core') return { ...actual, getActivities: vi.fn(), diff --git a/src/workflow/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts similarity index 98% rename from src/workflow/workflows/agent-embedding.ts rename to packages/agent/src/workflows/agent-embedding.ts index cdec798b..4cd38e46 100644 --- a/src/workflow/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -1,6 +1,6 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueDb, TaskQueueAgent } from '../workflow-utils' +import { getActivities, executeActivity, TaskQueueDb, TaskQueueAgent } from '@shumai/workflow-core' export async function agentEmbeddingMedia(task: WorkflowTask): Promise { const { diff --git a/src/workflow/workflows/agent-tool-call.ts b/packages/agent/src/workflows/agent-tool-call.ts similarity index 94% rename from src/workflow/workflows/agent-tool-call.ts rename to packages/agent/src/workflows/agent-tool-call.ts index 4ef5efbc..b7b7817f 100644 --- a/src/workflow/workflows/agent-tool-call.ts +++ b/packages/agent/src/workflows/agent-tool-call.ts @@ -1,6 +1,6 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueDb } from '../workflow-utils' +import { getActivities, executeActivity, TaskQueueDb } from '@shumai/workflow-core' export async function agentToolCall(task: WorkflowTask): Promise { const { updateTaskStatusActivity, executeAgentToolActivity } = getActivities() diff --git a/src/workflow/workflows/query-embedding-for-search.ts b/packages/agent/src/workflows/query-embedding-for-search.ts similarity index 94% rename from src/workflow/workflows/query-embedding-for-search.ts rename to packages/agent/src/workflows/query-embedding-for-search.ts index acc49dc3..2acc1a6c 100644 --- a/src/workflow/workflows/query-embedding-for-search.ts +++ b/packages/agent/src/workflows/query-embedding-for-search.ts @@ -1,5 +1,5 @@ import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueDb } from '../workflow-utils' +import { getActivities, executeActivity, TaskQueueDb } from '@shumai/workflow-core' export async function queryEmbeddingForSearch(task: WorkflowTask): Promise { const { generateTextEmbeddingActivity, updateWorkflowTaskActivity } = getActivities() diff --git a/packages/agent/tsconfig.json b/packages/agent/tsconfig.json new file mode 100644 index 00000000..fc4e681f --- /dev/null +++ b/packages/agent/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0" + }, + "include": ["src/**/*"] +} diff --git a/packages/core/src/search/search.test.ts b/packages/core/src/search/search.test.ts index 6833b743..026a3d38 100644 --- a/packages/core/src/search/search.test.ts +++ b/packages/core/src/search/search.test.ts @@ -4,7 +4,7 @@ import { setupTestDbHooks } from '@shumai/db' import { SearchService } from './search' import { SearchConditionOperator } from '@shumai/dtos' import { metadataService } from '@shumai/core/src/metadata/metadata' -import { workflowService } from '@/workflow/workflow' +import { workflowService } from '@shumai/workflow-core' import { AssetType } from '@shumai/db' diff --git a/packages/core/src/search/search.ts b/packages/core/src/search/search.ts index dcb0925c..f53bf48f 100644 --- a/packages/core/src/search/search.ts +++ b/packages/core/src/search/search.ts @@ -5,7 +5,7 @@ import { AssetInfo } from '@shumai/dtos' import { SearchRequest } from '@shumai/dtos' import { PaginatedData, decodeCursor, encodeCursor, PageInfo } from '@shumai/core/src/pagination' import { generateSearchNgrams } from '@/utils/ngram' -import { workflowService } from '@/workflow/workflow' +import { workflowService } from '@shumai/workflow-core' import { HTTPException } from 'hono/http-exception' import { SqlQueryBuilder } from './sql-query-builder' diff --git a/packages/core/src/upload/upload.ts b/packages/core/src/upload/upload.ts index 67c94f6a..d6fcb322 100644 --- a/packages/core/src/upload/upload.ts +++ b/packages/core/src/upload/upload.ts @@ -20,7 +20,7 @@ import { WorkflowTaskType, } from '@shumai/db' import { PaginationParams, paginateQuery, PaginatedData } from '@shumai/core/src/pagination' -import { VideoTranscoder, ImageTranscoder } from '@/transcode/transcoder' +import { VideoTranscoder, ImageTranscoder } from '@shumai/transcode' export class UploadService { constructor(private readonly prismaClient: typeof prisma = prisma) {} diff --git a/packages/transcode/package.json b/packages/transcode/package.json new file mode 100644 index 00000000..06c73dbb --- /dev/null +++ b/packages/transcode/package.json @@ -0,0 +1,18 @@ +{ + "name": "@shumai/transcode", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/core": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "@temporalio/activity": "^1.16.1", + "@temporalio/worker": "^1.16.1", + "@temporalio/workflow": "^1.16.1", + "sharp": "^0.34.5" + } +} diff --git a/src/transcode/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts similarity index 97% rename from src/transcode/activities/transcode.test.ts rename to packages/transcode/src/activities/transcode.test.ts index d12b241a..961fbe7f 100644 --- a/src/transcode/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -3,7 +3,7 @@ import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db' import { getMediaInfoActivity, transcodeVideoActivity, updateAssetMediaActivity } from './transcode' import { s3Service } from '@shumai/core/src/s3/s3' -import { transcodeService } from '@/transcode/transcode' +import { transcodeService } from '../transcode' vi.mock('@shumai/core/src/s3/s3', () => ({ s3Service: { @@ -14,7 +14,7 @@ vi.mock('@shumai/core/src/s3/s3', () => ({ }, })) -vi.mock('@/transcode/transcode', () => ({ +vi.mock('../transcode', () => ({ transcodeService: { getVideoInfo: vi.fn(), getImageInfo: vi.fn(), diff --git a/src/transcode/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts similarity index 99% rename from src/transcode/activities/transcode.ts rename to packages/transcode/src/activities/transcode.ts index 8679d921..f7a0dad6 100644 --- a/src/transcode/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -1,6 +1,6 @@ import { prisma } from '@shumai/db' import { s3Service } from '@shumai/core/src/s3/s3' -import { transcodeService } from '@/transcode/transcode' +import { transcodeService } from '../transcode' import { metadataService } from '@shumai/core/src/metadata/metadata' import { ApplicationFailure } from '@temporalio/activity' import * as path from 'path' diff --git a/packages/transcode/src/index.ts b/packages/transcode/src/index.ts new file mode 100644 index 00000000..3d89e822 --- /dev/null +++ b/packages/transcode/src/index.ts @@ -0,0 +1,14 @@ +import { registerWorkflow, registerActivities } from '@shumai/workflow-core' +import { WorkflowTaskType } from '@shumai/db' +import { transcodeMedia } from './workflows/transcode' +import * as transcodeActivities from './activities/transcode' + +export function initTranscodeWorkflows() { + registerWorkflow(WorkflowTaskType.transcode, transcodeMedia) + registerActivities(transcodeActivities) +} + +export * from './transcoder' +export * from './transcode' +export * from './workflows/transcode' +export * from './activities/transcode' diff --git a/src/transcode/transcode.test.ts b/packages/transcode/src/transcode.test.ts similarity index 100% rename from src/transcode/transcode.test.ts rename to packages/transcode/src/transcode.test.ts diff --git a/src/transcode/transcode.ts b/packages/transcode/src/transcode.ts similarity index 100% rename from src/transcode/transcode.ts rename to packages/transcode/src/transcode.ts diff --git a/src/transcode/transcoder.ts b/packages/transcode/src/transcoder.ts similarity index 100% rename from src/transcode/transcoder.ts rename to packages/transcode/src/transcoder.ts diff --git a/src/transcode/workflows/transcode.test.ts b/packages/transcode/src/workflows/transcode.test.ts similarity index 96% rename from src/transcode/workflows/transcode.test.ts rename to packages/transcode/src/workflows/transcode.test.ts index 2663aaa0..9dd8c2b0 100644 --- a/src/transcode/workflows/transcode.test.ts +++ b/packages/transcode/src/workflows/transcode.test.ts @@ -1,14 +1,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { transcodeMedia } from './transcode' -import { - WorkflowTask, - WorkflowTaskStatus, - WorkflowTaskType, - AssetStatus, -} from '@shumai/db' -import * as workflowUtils from '@/workflow/workflow-utils' +import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db' +import * as workflowUtils from '@shumai/workflow-core' -vi.mock('@/workflow/workflow-utils', async (importOriginal) => { +vi.mock('@shumai/workflow-core', async (importOriginal) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const actual = (await importOriginal()) as any return { diff --git a/src/transcode/workflows/transcode.ts b/packages/transcode/src/workflows/transcode.ts similarity index 99% rename from src/transcode/workflows/transcode.ts rename to packages/transcode/src/workflows/transcode.ts index 471d10fa..4f8d84dc 100644 --- a/src/transcode/workflows/transcode.ts +++ b/packages/transcode/src/workflows/transcode.ts @@ -5,7 +5,7 @@ import { executeActivity, TaskQueueDb, TaskQueueTranscode, -} from '@/workflow/workflow-utils' +} from '@shumai/workflow-core' export async function transcodeMedia(task: WorkflowTask): Promise { const { @@ -23,7 +23,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise { } = getActivities() let tmpDir: string | undefined - let workerQueue: string | undefined + let workerQueue = '' try { // 1. Update status to processing diff --git a/packages/transcode/tsconfig.json b/packages/transcode/tsconfig.json new file mode 100644 index 00000000..fc4e681f --- /dev/null +++ b/packages/transcode/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0" + }, + "include": ["src/**/*"] +} diff --git a/packages/worker-db/package.json b/packages/worker-db/package.json new file mode 100644 index 00000000..c5c3330b --- /dev/null +++ b/packages/worker-db/package.json @@ -0,0 +1,17 @@ +{ + "name": "@shumai/worker-db", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/core": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "@temporalio/activity": "^1.16.1", + "@temporalio/worker": "^1.16.1", + "@temporalio/workflow": "^1.16.1" + } +} diff --git a/src/workflow/activities/db.test.ts b/packages/worker-db/src/activities/db.test.ts similarity index 99% rename from src/workflow/activities/db.test.ts rename to packages/worker-db/src/activities/db.test.ts index bbbb5a1b..0b13068d 100644 --- a/src/workflow/activities/db.test.ts +++ b/packages/worker-db/src/activities/db.test.ts @@ -636,9 +636,10 @@ describe('Database Activities', () => { }) const updatedFile3 = await prisma.asset.findUnique({ where: { id: file3Result.id } }) + const freshFile2 = await prisma.asset.findUnique({ where: { id: file2Result.id } }) // Newest version (file3) must have a strictly lower sortIndex than version 2 (file2) - expect(updatedFile3!.sortIndex! < updatedFile2!.sortIndex!).toBe(true) + expect(updatedFile3!.sortIndex! < freshFile2!.sortIndex!).toBe(true) }) }) }) diff --git a/src/workflow/activities/db.ts b/packages/worker-db/src/activities/db.ts similarity index 99% rename from src/workflow/activities/db.ts rename to packages/worker-db/src/activities/db.ts index b4eebab0..67c0c98b 100644 --- a/src/workflow/activities/db.ts +++ b/packages/worker-db/src/activities/db.ts @@ -5,7 +5,7 @@ import { ulid } from 'ulid' import { metadataService } from '@shumai/core/src/metadata/metadata' import { UpdateAssetMetadataRequest } from '@shumai/dtos' import { WorkflowTaskStatus, AssetStatus, Prisma, AssetType } from '@shumai/db' -import type { AgentExecutionContext } from './agent' +import type { AgentExecutionContext } from '@shumai/agent' import { assetService } from '@shumai/core/src/asset/asset' import { VersionStackService } from '@shumai/core/src/versionStack/versionStack' import { uploadService } from '@shumai/core/src/upload/upload' diff --git a/packages/worker-db/src/index.ts b/packages/worker-db/src/index.ts new file mode 100644 index 00000000..b42351dd --- /dev/null +++ b/packages/worker-db/src/index.ts @@ -0,0 +1,8 @@ +import { registerActivities } from '@shumai/workflow-core' +import * as dbActivities from './activities/db' + +export function initDbWorkflows() { + registerActivities(dbActivities) +} + +export * from './activities/db' diff --git a/packages/worker-db/tsconfig.json b/packages/worker-db/tsconfig.json new file mode 100644 index 00000000..fc4e681f --- /dev/null +++ b/packages/worker-db/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0" + }, + "include": ["src/**/*"] +} diff --git a/packages/workflow-core/package.json b/packages/workflow-core/package.json new file mode 100644 index 00000000..f0f42785 --- /dev/null +++ b/packages/workflow-core/package.json @@ -0,0 +1,15 @@ +{ + "name": "@shumai/workflow-core", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "dependencies": { + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/core": "workspace:*", + "@temporalio/client": "^1.16.1", + "@temporalio/workflow": "^1.16.1" + } +} diff --git a/src/workflow/activities/task.test.ts b/packages/workflow-core/src/activities/task.test.ts similarity index 94% rename from src/workflow/activities/task.test.ts rename to packages/workflow-core/src/activities/task.test.ts index 44aa1e24..51526047 100644 --- a/src/workflow/activities/task.test.ts +++ b/packages/workflow-core/src/activities/task.test.ts @@ -1,7 +1,11 @@ import { describe, it, expect } from 'vitest' import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db' -import { updateTaskStatusActivity, updateAssetStatusActivity, updateTaskUsageActivity } from './db' +import { + updateTaskStatusActivity, + updateAssetStatusActivity, + updateTaskUsageActivity, +} from '@shumai/worker-db' import { WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db' describe('Task Activities', () => { diff --git a/src/workflow/activities/task.ts b/packages/workflow-core/src/activities/task.ts similarity index 100% rename from src/workflow/activities/task.ts rename to packages/workflow-core/src/activities/task.ts diff --git a/src/workflow/executor.ts b/packages/workflow-core/src/executor.ts similarity index 100% rename from src/workflow/executor.ts rename to packages/workflow-core/src/executor.ts diff --git a/packages/workflow-core/src/index.ts b/packages/workflow-core/src/index.ts new file mode 100644 index 00000000..ef52a548 --- /dev/null +++ b/packages/workflow-core/src/index.ts @@ -0,0 +1,6 @@ +export * from './executor' +export * from './local-executor' +export * from './temporal-executor' +export * from './workflow-utils' +export * from './workflow' +export * from './activities/task' diff --git a/src/workflow/local-executor.test.ts b/packages/workflow-core/src/local-executor.test.ts similarity index 94% rename from src/workflow/local-executor.test.ts rename to packages/workflow-core/src/local-executor.test.ts index 04965012..8892d86b 100644 --- a/src/workflow/local-executor.test.ts +++ b/packages/workflow-core/src/local-executor.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest' import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db' import { workflowService } from './workflow' -import { LocalExecutor, ConcurrencyLimiter } from './local-executor' +import { LocalExecutor, ConcurrencyLimiter, registerWorkflow } from './local-executor' import { WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' // Mock the individual workflows @@ -14,22 +14,6 @@ const mocks = { transcodeMedia: vi.fn(), } -vi.mock('./workflows/agent-embedding', () => ({ - agentEmbeddingMedia: (task: unknown) => mocks.agentEmbeddingMedia(task), -})) -vi.mock('./workflows/query-embedding-for-search', () => ({ - queryEmbeddingForSearch: (task: unknown) => mocks.queryEmbeddingForSearch(task), -})) -vi.mock('./workflows/agent-autofill', () => ({ - agentAutofillMedia: (task: unknown) => mocks.agentAutofillMedia(task), -})) -vi.mock('./workflows/agent-chat', () => ({ - agentChat: (task: unknown) => mocks.agentChat(task), -})) -vi.mock('@/transcode/workflows/transcode', () => ({ - transcodeMedia: (task: unknown) => mocks.transcodeMedia(task), -})) - describe('LocalExecutor Integration Tests', () => { setupTestDbHooks() @@ -39,6 +23,17 @@ describe('LocalExecutor Integration Tests', () => { // Get the global executor instance which processes tasks submitted via the Prisma Client Extension executor = (workflowService as unknown as { executor: LocalExecutor }).executor vi.clearAllMocks() + + // Register workflows dynamically for testing + registerWorkflow(WorkflowTaskType.ai_embedding, async (task) => mocks.agentEmbeddingMedia(task)) + registerWorkflow(WorkflowTaskType.query_embedding_for_search, async (task) => + mocks.queryEmbeddingForSearch(task), + ) + registerWorkflow(WorkflowTaskType.ai_metadata_autofill, async (task) => + mocks.agentAutofillMedia(task), + ) + registerWorkflow(WorkflowTaskType.chat, async (task) => mocks.agentChat(task)) + registerWorkflow(WorkflowTaskType.transcode, async (task) => mocks.transcodeMedia(task)) }) describe('Heartbeat & Task Status transition', () => { diff --git a/src/workflow/local-executor.ts b/packages/workflow-core/src/local-executor.ts similarity index 84% rename from src/workflow/local-executor.ts rename to packages/workflow-core/src/local-executor.ts index 33a099ae..3a1a7e07 100644 --- a/src/workflow/local-executor.ts +++ b/packages/workflow-core/src/local-executor.ts @@ -1,20 +1,33 @@ import { prisma } from '@shumai/db' import { WorkflowTask, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' import { Executor } from './executor' -import { agentEmbeddingMedia } from './workflows/agent-embedding' -import { queryEmbeddingForSearch } from './workflows/query-embedding-for-search' -import { agentAutofillMedia } from './workflows/agent-autofill' -import { agentChat } from './workflows/agent-chat' -import { transcodeMedia } from '@/transcode/workflows/transcode' -import { agentToolCall } from './workflows/agent-tool-call' -import { activities } from './activities/index' +import * as taskActivities from './activities/task' import { logger } from '@/logger' +type WorkflowFn = (task: WorkflowTask) => Promise + +const workflowRegistry = new Map() +const activityRegistry: Record = { ...taskActivities } + +export function registerWorkflow(type: string, fn: WorkflowFn) { + workflowRegistry.set(type, fn) +} + +/* eslint-disable @typescript-eslint/naming-convention */ +const globalObj = globalThis as typeof globalThis & { + __localActivities?: Record + __localLogger?: unknown +} +/* eslint-enable @typescript-eslint/naming-convention */ + +export function registerActivities(acts: Record) { + Object.assign(activityRegistry, acts) + globalObj.__localActivities = activityRegistry +} + // Inject local dependencies into global scope for workflow-utils -// eslint-disable-next-line @typescript-eslint/no-explicit-any -;(globalThis as any).__localActivities = activities -// eslint-disable-next-line @typescript-eslint/no-explicit-any -;(globalThis as any).__localLogger = logger +globalObj.__localActivities = activityRegistry +globalObj.__localLogger = logger export class ConcurrencyLimiter { private activeCount = 0 @@ -235,27 +248,11 @@ export class LocalExecutor implements Executor { private async processTaskWrapper(task: WorkflowTask) { try { - switch (task.type) { - case WorkflowTaskType.ai_embedding: - await agentEmbeddingMedia(task) - break - case WorkflowTaskType.query_embedding_for_search: - await queryEmbeddingForSearch(task) - break - case WorkflowTaskType.ai_metadata_autofill: - await agentAutofillMedia(task) - break - case WorkflowTaskType.chat: - await agentChat(task) - break - case WorkflowTaskType.transcode: - await transcodeMedia(task) - break - case WorkflowTaskType.agent_tool_call: - await agentToolCall(task) - break - default: - console.warn(`Unknown task type: ${task.type}`) + const fn = task.type ? workflowRegistry.get(task.type) : undefined + if (fn) { + await fn(task) + } else { + console.warn(`Unknown or unregistered task type: ${task.type}`) } } catch (err) { console.error(`Error in workflow ${task.type} for task ${task.id}:`, err) diff --git a/src/workflow/temporal-executor.ts b/packages/workflow-core/src/temporal-executor.ts similarity index 100% rename from src/workflow/temporal-executor.ts rename to packages/workflow-core/src/temporal-executor.ts diff --git a/src/workflow/workflow-utils.test.ts b/packages/workflow-core/src/workflow-utils.test.ts similarity index 100% rename from src/workflow/workflow-utils.test.ts rename to packages/workflow-core/src/workflow-utils.test.ts diff --git a/src/workflow/workflow-utils.ts b/packages/workflow-core/src/workflow-utils.ts similarity index 68% rename from src/workflow/workflow-utils.ts rename to packages/workflow-core/src/workflow-utils.ts index 3521deff..dd61acac 100644 --- a/src/workflow/workflow-utils.ts +++ b/packages/workflow-core/src/workflow-utils.ts @@ -1,5 +1,4 @@ // Use type-only imports for activities to avoid bundling them in Temporal workflows -import type { activities } from './activities/index' import * as wf from '@temporalio/workflow' /** @@ -22,12 +21,25 @@ export function isTemporal(): boolean { } } +interface LocalLogger { + debug(obj: Record, msg: string): void + info(obj: Record, msg: string): void + error(obj: Record, msg: string): void + warn(obj: Record, msg: string): void +} + +/* eslint-disable @typescript-eslint/naming-convention */ +const globalObj = globalThis as typeof globalThis & { + __localActivities?: Record + __localLogger?: LocalLogger +} +/* eslint-enable @typescript-eslint/naming-convention */ + /** * A proxy that mimics proxyActivities for local execution. */ -function localActivitiesProxy(): typeof activities { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const acts = (globalThis as any).__localActivities +function localActivitiesProxy(): Record { + const acts = globalObj.__localActivities if (!acts) { throw new Error('Local activities not injected into globalThis.__localActivities') } @@ -48,11 +60,17 @@ export async function sleep(ms: number): Promise { /** * Environment-aware activity proxy. */ -export function getActivities(): typeof activities { +// getActivities must default to `any` so that workflows can dynamically +// destructure and call any activity when they do not supply a precise type parameter. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function getActivities(): T { + // baseProxy is dynamically proxying all activities, so we must type it as any. // eslint-disable-next-line @typescript-eslint/no-explicit-any let baseProxy: any if (isTemporal()) { - baseProxy = wf.proxyActivities({ + // Temporal's proxyActivities requires a type parameter, so we use any for generic activities. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + baseProxy = wf.proxyActivities({ startToCloseTimeout: '10 minutes', }) } else { @@ -93,8 +111,7 @@ function getLogger() { warn: (obj: Record, msg: string) => wf.log.warn(msg, obj), } } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return (globalThis as any).__localLogger || console + return globalObj.__localLogger || console } /** @@ -103,11 +120,29 @@ function getLogger() { * @param activityFunc The proxy function obtained from `getActivities()` * @param args The arguments to pass to the activity */ +// Overload signature for untyped or generic activity callbacks. +export async function executeActivity( + queue: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + activityFunc: any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...args: any[] + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): Promise export async function executeActivity( queue: string, activityFunc: (...args: TArgs) => Promise, ...args: TArgs -): Promise { +): Promise +// Implementation signature for executeActivity. +export async function executeActivity( + queue: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + activityFunc: any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...args: any[] + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): Promise { // We extract the hidden _activityName property attached by getActivities() // eslint-disable-next-line @typescript-eslint/no-explicit-any const activityName = (activityFunc as any)._activityName @@ -121,7 +156,9 @@ export async function executeActivity( logger.debug({ queue, activityName }, `Executing activity ${activityName} on queue ${queue}`) try { - let result: TRet + // result holds the untyped return value from the dynamic activity proxy, so we type it as any. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let result: any if (isTemporal()) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const handle = wf.proxyActivities({ diff --git a/src/workflow/workflow.test.ts b/packages/workflow-core/src/workflow.test.ts similarity index 87% rename from src/workflow/workflow.test.ts rename to packages/workflow-core/src/workflow.test.ts index 237082b8..16d471d3 100644 --- a/src/workflow/workflow.test.ts +++ b/packages/workflow-core/src/workflow.test.ts @@ -2,6 +2,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { prisma } from '@shumai/db' import { workflowService } from './workflow' import { WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' +import { initAgentWorkflows } from '@shumai/agent' +import { initTranscodeWorkflows } from '@shumai/transcode' +import { initDbWorkflows } from '@shumai/worker-db' vi.mock('@shumai/core/src/s3/s3', () => ({ s3Service: { @@ -13,22 +16,29 @@ vi.mock('@shumai/core/src/s3/s3', () => ({ }, })) -vi.mock('@/transcode/transcode', () => ({ - transcodeService: { - createTempDir: vi.fn().mockReturnValue('/tmp/shumai-test'), - removeDir: vi.fn(), - extractAudio: vi.fn(), - extractVideoFrames: vi.fn().mockResolvedValue([]), - getVideoInfo: vi.fn(), - getImageInfo: vi.fn(), - transcodeVideo: vi.fn(), - transcodeImage: vi.fn(), - generateSprite: vi.fn(), - }, -})) +vi.mock('@shumai/transcode', async (importOriginal) => { + const actual = (await importOriginal()) as Record + return { + ...actual, + transcodeService: { + createTempDir: vi.fn().mockReturnValue('/tmp/shumai-test'), + removeDir: vi.fn(), + extractAudio: vi.fn(), + extractVideoFrames: vi.fn().mockResolvedValue([]), + getVideoInfo: vi.fn(), + getImageInfo: vi.fn(), + transcodeVideo: vi.fn(), + transcodeImage: vi.fn(), + generateSprite: vi.fn(), + }, + } +}) describe('WorkflowService', () => { beforeEach(async () => { + initAgentWorkflows() + initTranscodeWorkflows() + initDbWorkflows() await prisma.workflowTask.deleteMany() await prisma.assetComment.deleteMany() await prisma.asset.deleteMany() diff --git a/src/workflow/workflow.ts b/packages/workflow-core/src/workflow.ts similarity index 88% rename from src/workflow/workflow.ts rename to packages/workflow-core/src/workflow.ts index 015b8748..6da227fd 100644 --- a/src/workflow/workflow.ts +++ b/packages/workflow-core/src/workflow.ts @@ -56,18 +56,26 @@ export class WorkflowService { } // Starts Temporal workers for specific task queues - async startWorkers(queue: string = 'db_queue'): Promise { + async startWorkers( + queue: string = 'db_queue', + options?: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + workflowBundle?: any + // eslint-disable-next-line @typescript-eslint/no-explicit-any + activities?: any + }, + ): Promise { const type = process.env.WORKFLOW_EXECUTOR || 'local' if (type === 'temporal') { const { Worker, NativeConnection } = await import('@temporalio/worker') - const { activities } = await import('./activities/index') const connection = await NativeConnection.connect({ address: process.env.TEMPORAL_ADDRESS || 'localhost:7233', }) - // @ts-ignore - const { default: workflowBundle } = await import('./workflows:::workflow') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const activities = options?.activities || (globalThis as any).__localActivities + const workflowBundle = options?.workflowBundle if (queue === 'db_queue') { console.log(`Starting Temporal worker for queue: ${queue}`) diff --git a/packages/workflow-core/tsconfig.json b/packages/workflow-core/tsconfig.json new file mode 100644 index 00000000..fc4e681f --- /dev/null +++ b/packages/workflow-core/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "ignoreDeprecations": "6.0" + }, + "include": ["src/**/*"] +} diff --git a/src/cli/serve.ts b/src/cli/serve.ts index ab071152..f3b17df8 100644 --- a/src/cli/serve.ts +++ b/src/cli/serve.ts @@ -24,7 +24,10 @@ import uploadRoute from '@/api/upload' import versionStackRoute from '@/api/versionStack' import { metadataService } from '@shumai/core/src/metadata/metadata' import { assetService } from '@shumai/core/src/asset/asset' -import { workflowService } from '@/workflow/workflow' +import { workflowService } from '@shumai/workflow-core' +import { initAgentWorkflows } from '@shumai/agent' +import { initTranscodeWorkflows } from '@shumai/transcode' +import { initDbWorkflows } from '@shumai/worker-db' type User = Prisma.UserGetPayload> @@ -62,6 +65,11 @@ const apiAppRoute = app.route('/api', apiRoute) export type AppType = typeof apiAppRoute export async function run() { + // Initialize workflows and activities for local executor mode + initAgentWorkflows() + initTranscodeWorkflows() + initDbWorkflows() + // Start services await metadataService.syncSystemFields().catch(console.error) assetService.startCleanupJob() diff --git a/src/cli/worker.ts b/src/cli/worker.ts index bafc75ba..53a29509 100644 --- a/src/cli/worker.ts +++ b/src/cli/worker.ts @@ -1,7 +1,15 @@ -import { workflowService } from '@/workflow/workflow' -import { TaskQueueAgent, TaskQueueTranscode } from '@/workflow/workflow-utils' +import { workflowService } from '@shumai/workflow-core' +import { TaskQueueAgent, TaskQueueTranscode } from '@shumai/workflow-core' +import { initAgentWorkflows } from '@shumai/agent' +import { initTranscodeWorkflows } from '@shumai/transcode' +import { initDbWorkflows } from '@shumai/worker-db' export async function run() { + // Initialize workflows and activities + initAgentWorkflows() + initTranscodeWorkflows() + initDbWorkflows() + const domain = process.argv[3] if (!domain) { diff --git a/src/workflow/activities/index.ts b/src/workflow/activities/index.ts deleted file mode 100644 index 0e8028d9..00000000 --- a/src/workflow/activities/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as transcodeActivities from '@/transcode/activities/transcode' -import * as agentActivities from './agent' -import * as aiActivities from './ai' -import * as dbActivities from './db' -import * as taskActivities from './task' - -export const activities = { - ...taskActivities, - ...aiActivities, - ...transcodeActivities, - ...agentActivities, - ...dbActivities, -} diff --git a/src/workflow/workflows/index.ts b/src/workflow/workflows/index.ts deleted file mode 100644 index 691bdf18..00000000 --- a/src/workflow/workflows/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './agent-embedding' -export * from './agent-autofill' -export * from './agent-chat' -export * from './query-embedding-for-search' -export * from './agent-tool-call' -export * from '@/transcode/workflows/transcode'