mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 15:28:26 +00:00
feat(workflow): integrate temporal workflow bundles and support dev/prod mode
This commit is contained in:
@@ -113,15 +113,15 @@ The backend is built with:
|
||||
|
||||
The project is a monorepo managed by **Bun Workspaces**. It follows a strictly decoupled architecture where each domain or layer is its own package:
|
||||
|
||||
- **WebUI (`packages/webui`)**: React-based frontend.
|
||||
- **API (`packages/api`)**: Hono-based HTTP entry point. Handles requests and calls Core services.
|
||||
- **Core (`packages/core`)**: Business logic, services, and infrastructure utilities.
|
||||
- **Database (`packages/db`)**: Prisma client, schema, and migrations.
|
||||
- **DTOs (`packages/dtos`)**: Shared type definitions and Zod schemas used by both API and WebUI.
|
||||
- **Workers**: Specialized packages for background task execution:
|
||||
- `@shumai/workflow-core`: Common workflow engine logic.
|
||||
- `@shumai/agent`: AI agent workflows and activities.
|
||||
- `@shumai/transcode`: Media processing workflows and activities.
|
||||
- **WebUI (`packages/webui`)**: React-based frontend.
|
||||
- **API (`packages/api`)**: Hono-based HTTP entry point. Handles requests and calls Core services.
|
||||
- **Core (`packages/core`)**: Business logic, services, and infrastructure utilities.
|
||||
- **Database (`packages/db`)**: Prisma client, schema, and migrations.
|
||||
- **DTOs (`packages/dtos`)**: Shared type definitions and Zod schemas used by both API and WebUI.
|
||||
- **Workers**: Specialized packages for background task execution:
|
||||
- `@shumai/workflow-core`: Common workflow engine logic.
|
||||
- `@shumai/agent`: AI agent workflows and activities.
|
||||
- `@shumai/transcode`: Media processing workflows and activities.
|
||||
|
||||
### Layered Communication Rules
|
||||
|
||||
|
||||
+11
-1
@@ -10,7 +10,17 @@ async function run() {
|
||||
initAgentWorkflows()
|
||||
|
||||
console.log('🚀 Starting Agent Worker...')
|
||||
await workflowService.startWorkers(TaskQueueAgent)
|
||||
|
||||
const options: { workflowBundle?: unknown; workflowsPath?: string } = {}
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
// @ts-ignore: Bun temporal plugin macro
|
||||
const { default: workflowBundle } = await import('./workflows:::workflow')
|
||||
options.workflowBundle = workflowBundle
|
||||
} else {
|
||||
options.workflowsPath = new URL('./workflows.ts', import.meta.url).pathname
|
||||
}
|
||||
|
||||
await workflowService.startWorkers(TaskQueueAgent, options)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from '@shumai/agent/src/workflows/agent-autofill'
|
||||
export * from '@shumai/agent/src/workflows/agent-chat'
|
||||
export * from '@shumai/agent/src/workflows/agent-embedding'
|
||||
export * from '@shumai/agent/src/workflows/agent-tool-call'
|
||||
export * from '@shumai/agent/src/workflows/query-embedding-for-search'
|
||||
@@ -1,16 +1,25 @@
|
||||
import { loadEnvConfig } from '@shumai/core/src/env-loader'
|
||||
loadEnvConfig(process.cwd(), process.env.NODE_ENV !== 'production')
|
||||
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { TaskQueueTranscode } from '@shumai/workflow-core'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { TaskQueueTranscode, workflowService } from '@shumai/workflow-core'
|
||||
|
||||
async function run() {
|
||||
// Initialize workflows and activities
|
||||
initTranscodeWorkflows()
|
||||
|
||||
console.log('🚀 Starting Transcode Worker...')
|
||||
await workflowService.startWorkers(TaskQueueTranscode)
|
||||
|
||||
const options: { workflowBundle?: unknown; workflowsPath?: string } = {}
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
// @ts-ignore: Bun temporal plugin macro
|
||||
const { default: workflowBundle } = await import('./workflows:::workflow')
|
||||
options.workflowBundle = workflowBundle
|
||||
} else {
|
||||
options.workflowsPath = new URL('./workflows.ts', import.meta.url).pathname
|
||||
}
|
||||
|
||||
await workflowService.startWorkers(TaskQueueTranscode, options)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from '@shumai/transcode/src/workflows/transcode'
|
||||
@@ -3,7 +3,11 @@ import { DatabaseSessionStorage } from '../database-session-storage'
|
||||
import { prisma, AssetType, Prisma, type Skill } from '@shumai/db'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { type AgentTool, type AgentMessage, type SessionTreeEntry } from '@earendil-works/pi-agent-core'
|
||||
import {
|
||||
type AgentTool,
|
||||
type AgentMessage,
|
||||
type SessionTreeEntry,
|
||||
} from '@earendil-works/pi-agent-core'
|
||||
import { type ImageContent } from '@earendil-works/pi-ai'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import { assetService } from '@shumai/core/src/asset/asset'
|
||||
|
||||
@@ -261,10 +261,7 @@ export async function extractAiMetadataActivity(
|
||||
return generatedFiles
|
||||
}
|
||||
|
||||
export async function getEmbeddingContextActivity(params: {
|
||||
teamId: string
|
||||
assetId: string
|
||||
}) {
|
||||
export async function getEmbeddingContextActivity(params: { teamId: string; assetId: string }) {
|
||||
const team = await prisma.team.findUnique({
|
||||
where: { id: params.teamId },
|
||||
})
|
||||
|
||||
@@ -93,7 +93,11 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
|
||||
// 3. Get Project Autofill Fields
|
||||
const fields = await executeActivity(TaskQueueAgent, getProjectAutofillFieldsActivity, projectId)
|
||||
const fields = await executeActivity(
|
||||
TaskQueueAgent,
|
||||
getProjectAutofillFieldsActivity,
|
||||
projectId,
|
||||
)
|
||||
if (fields.length === 0) {
|
||||
if (placeholderCommentId) {
|
||||
await executeActivity(TaskQueueAgent, updateCommentActivity, {
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
import { registerWorkflow, registerActivities } from '@shumai/workflow-core'
|
||||
import { WorkflowTaskType } from '@shumai/db'
|
||||
import { transcodeMedia } from './workflows/transcode'
|
||||
import { registerActivities, registerWorkflow } from '@shumai/workflow-core'
|
||||
import * as transcodeActivities from './activities/transcode'
|
||||
import { transcodeMedia } from './workflows/transcode'
|
||||
|
||||
export function initTranscodeWorkflows() {
|
||||
registerWorkflow(WorkflowTaskType.transcode, transcodeMedia)
|
||||
registerActivities(transcodeActivities)
|
||||
}
|
||||
|
||||
export * from './transcoder'
|
||||
export * from './transcode'
|
||||
export * from './workflows/transcode'
|
||||
export * from './activities/transcode'
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import {
|
||||
getActivities,
|
||||
executeActivity,
|
||||
TaskQueueTranscode,
|
||||
} from '@shumai/workflow-core'
|
||||
executeActivity,
|
||||
getActivities,
|
||||
TaskQueueTranscode,
|
||||
} from '@shumai/workflow-core/src/workflow-utils'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
|
||||
export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
const {
|
||||
|
||||
@@ -9,7 +9,9 @@ export async function getAgentWorkerQueueActivity(): Promise<string> {
|
||||
return 'agent_queue'
|
||||
}
|
||||
|
||||
export async function getAssetStatusActivity(params: { assetId: string }): Promise<AssetStatus | null> {
|
||||
export async function getAssetStatusActivity(params: {
|
||||
assetId: string
|
||||
}): Promise<AssetStatus | null> {
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: params.assetId },
|
||||
select: { status: true },
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { WorkflowTask } from '@shumai/db'
|
||||
import { prisma, registerWorkflowTrigger, WorkflowTask } from '@shumai/db'
|
||||
import path from 'path'
|
||||
import { Executor } from './executor'
|
||||
import { LocalExecutor } from './local-executor'
|
||||
import { TemporalExecutor } from './temporal-executor'
|
||||
import { prisma, registerWorkflowTrigger } from '@shumai/db'
|
||||
|
||||
export class WorkflowService {
|
||||
private executor: Executor
|
||||
@@ -56,7 +56,10 @@ export class WorkflowService {
|
||||
}
|
||||
|
||||
// Starts Temporal workers for specific task queues
|
||||
async startWorkers(queue: string): Promise<void> {
|
||||
async startWorkers(
|
||||
queue: string,
|
||||
options: { workflowBundle?: unknown; workflowsPath?: string } = {},
|
||||
): Promise<void> {
|
||||
const type = process.env.WORKFLOW_EXECUTOR || 'local'
|
||||
if (type === 'temporal') {
|
||||
const { Worker, NativeConnection } = await import('@temporalio/worker')
|
||||
@@ -85,10 +88,31 @@ export class WorkflowService {
|
||||
sharedActivities.getTranscodeWorkerQueueActivity = async () => workerSpecificQueue
|
||||
}
|
||||
|
||||
const { workflowBundle, workflowsPath } = options
|
||||
|
||||
const sharedWorker = await Worker.create({
|
||||
connection,
|
||||
// The workflowBundle is a pre-bundled set of workflows, its internal structure
|
||||
// is managed by the Temporal worker and doesn't need explicit typing here.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
...(workflowBundle ? { workflowBundle: workflowBundle as any } : {}),
|
||||
...(workflowsPath ? { workflowsPath } : {}),
|
||||
activities: sharedActivities,
|
||||
taskQueue: queue,
|
||||
|
||||
bundlerOptions: {
|
||||
webpackConfigHook: (config) => {
|
||||
config.resolve = config.resolve || {}
|
||||
config.resolve.alias = {
|
||||
...config.resolve.alias,
|
||||
// Find the exact absolute path to the package directory
|
||||
'@temporalio/workflow': path.dirname(
|
||||
require.resolve('@temporalio/workflow/package.json'),
|
||||
),
|
||||
}
|
||||
return config
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const specificWorker = await Worker.create({
|
||||
@@ -96,6 +120,19 @@ export class WorkflowService {
|
||||
// No workflows needed for the unique queue
|
||||
activities,
|
||||
taskQueue: workerSpecificQueue,
|
||||
bundlerOptions: {
|
||||
webpackConfigHook: (config) => {
|
||||
config.resolve = config.resolve || {}
|
||||
config.resolve.alias = {
|
||||
...config.resolve.alias,
|
||||
// Find the exact absolute path to the package directory
|
||||
'@temporalio/workflow': path.dirname(
|
||||
require.resolve('@temporalio/workflow/package.json'),
|
||||
),
|
||||
}
|
||||
return config
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await Promise.all([sharedWorker.run(), specificWorker.run()])
|
||||
|
||||
+4
-1
@@ -21,8 +21,11 @@
|
||||
"@shumai/db/test": ["packages/db/src/test.ts"],
|
||||
"@shumai/dtos": ["packages/dtos/src/index.ts"],
|
||||
"@shumai/workflow-core": ["packages/workflow-core/src/index.ts"],
|
||||
"@shumai/workflow-core/src/*": ["packages/workflow-core/src/*"],
|
||||
"@shumai/agent": ["packages/agent/src/index.ts"],
|
||||
"@shumai/transcode": ["packages/transcode/src/index.ts"]
|
||||
"@shumai/agent/src/*": ["packages/agent/src/*"],
|
||||
"@shumai/transcode": ["packages/transcode/src/index.ts"],
|
||||
"@shumai/transcode/src/*": ["packages/transcode/src/*"]
|
||||
}
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
|
||||
Reference in New Issue
Block a user