From 4264849f36a7e73f1eaef905b8b18c94b839bdbe Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 3 Jun 2026 09:21:51 +0800 Subject: [PATCH] feat(workflow): strictly enforce queue redirection and activity registration rules --- AGENTS.md | 4 ++ .../agent/src/workflows/agent-autofill.ts | 58 ++++++++++--------- packages/agent/src/workflows/agent-chat.ts | 44 +++++++------- .../agent/src/workflows/agent-embedding.ts | 36 +++++++----- packages/transcode/src/index.ts | 5 ++ packages/transcode/src/workflows/transcode.ts | 36 ++++++------ packages/workflow-core/src/workflow.ts | 5 +- 7 files changed, 105 insertions(+), 83 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0dcd02bf..403e2c95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -380,6 +380,10 @@ To prevent Temporal from indefinitely retrying fatal, expected business validati ``` - **Environment Compatibility**: Always use `getActivities()` from `@shumai/workflow-core` to ensure the code works in both Local and Temporal environments. - **Activity Access**: Activities should be accessed via `getActivities()` within a workflow. Do not call services directly inside a workflow function to maintain Temporal compatibility. +- **Queue Redirection**: Production Temporal workers use worker-specific unique queues for data locality (e.g., local disk access) and consistent state. + - Every workflow MUST first call `getAgentWorkerQueueActivity` or `getTranscodeWorkerQueueActivity` via the shared domain queue (e.g., `TaskQueueAgent` or `TaskQueueTranscode`) to obtain the specific queue name for that worker instance. + - ALL subsequent activities (including shared DB activities like updating task status) MUST be executed on that specific discovered queue. + - The shared domain worker ONLY registers the queue discovery activities. All other activities (shared and domain-specific) are registered on the specific worker. ### Activity Patterns diff --git a/packages/agent/src/workflows/agent-autofill.ts b/packages/agent/src/workflows/agent-autofill.ts index 2ec1dda5..340fa09f 100644 --- a/packages/agent/src/workflows/agent-autofill.ts +++ b/packages/agent/src/workflows/agent-autofill.ts @@ -28,17 +28,25 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { let placeholderCommentId: string | undefined let tmpDir: string | undefined let transcodeWorkerQueue = '' + let agentWorkerQueue = '' try { + // 0. Discover queues + agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) + transcodeWorkerQueue = await executeActivity( + TaskQueueTranscode, + getTranscodeWorkerQueueActivity, + ) + // Update status to processing - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) // 0. Create Placeholder Comment const payload = task.payload - const placeholder = await executeActivity(TaskQueueAgent, createCommentActivity, { + const placeholder = await executeActivity(agentWorkerQueue, createCommentActivity, { assetId: task.assetId, message: '__AUTOFILL__', sessionId: payload?.agent?.sessionId || task.id, @@ -47,7 +55,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { placeholderCommentId = placeholder.id // 1. Get Asset - const asset = await executeActivity(TaskQueueAgent, getAssetActivity, task.assetId) + const asset = await executeActivity(agentWorkerQueue, getAssetActivity, task.assetId) const key = asset?.storageKey?.key if (!asset || !asset.project || !key) { throw ApplicationFailure.create({ message: 'Asset or project not found', nonRetryable: true }) @@ -58,12 +66,6 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { // 2. Prepare Data (Images) const isImage = asset.mediaType?.startsWith('image/') || false - // Extraction requires FFmpeg, run on Transcode worker - transcodeWorkerQueue = await executeActivity( - TaskQueueTranscode, - getTranscodeWorkerQueueActivity, - ) - const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, { assetKey: key, }) @@ -80,12 +82,12 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { if (generatedFiles.length === 0) { // If no images could be extracted, we can't do much if (placeholderCommentId) { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: 'Autofill completed: No images could be extracted for analysis.', }) } - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -94,18 +96,18 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { // 3. Get Project Autofill Fields const fields = await executeActivity( - TaskQueueAgent, + agentWorkerQueue, getProjectAutofillFieldsActivity, projectId, ) if (fields.length === 0) { if (placeholderCommentId) { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: 'Autofill completed: No autofill fields defined in project.', }) } - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -113,13 +115,11 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { } // 3b. Fetch Agent Context - const context = await executeActivity(TaskQueueAgent, getAgentAutofillContextActivity, { + const context = await executeActivity(agentWorkerQueue, getAgentAutofillContextActivity, { teamId, }) // 4. Call AI Service - const agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) - const aiResult = await executeActivity(agentWorkerQueue, autofillAiActivity, { teamId, images: generatedFiles, @@ -135,7 +135,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { if (aiResult.usage) { // Update Usage - await executeActivity(TaskQueueAgent, updateTaskUsageActivity, { + await executeActivity(agentWorkerQueue, updateTaskUsageActivity, { taskId: task.id, inputTokens: aiResult.usage.inputTokens, outputTokens: aiResult.usage.outputTokens, @@ -151,7 +151,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { })) if (metadataUpdates.length > 0) { - await executeActivity(TaskQueueAgent, updateAssetMetadataActivity, { + await executeActivity(agentWorkerQueue, updateAssetMetadataActivity, { assetId: asset.id, metadata: metadataUpdates, }) @@ -159,7 +159,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { // 7. Update Placeholder Comment if (placeholderCommentId) { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: 'Autofill completed successfully.', sessionId: aiResult.sessionId, @@ -167,7 +167,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { } // Update status to completed - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -175,9 +175,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { console.error(`AgentAutofillMedia failed for task ${task.id}:`, err) // Update placeholder comment with error message - if (placeholderCommentId) { + if (placeholderCommentId && agentWorkerQueue) { try { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: `Autofill failed: ${err instanceof Error ? err.message : String(err)}`, }) @@ -187,11 +187,13 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { } // Update status to failed - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { - taskId: task.id, - status: 'failed', - output: { error: err instanceof Error ? err.message : String(err) }, - }) + if (agentWorkerQueue) { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { + taskId: task.id, + status: 'failed', + output: { error: err instanceof Error ? err.message : String(err) }, + }) + } throw err } finally { if (tmpDir && transcodeWorkerQueue) { diff --git a/packages/agent/src/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts index 216bd82c..9271d470 100644 --- a/packages/agent/src/workflows/agent-chat.ts +++ b/packages/agent/src/workflows/agent-chat.ts @@ -19,10 +19,14 @@ export async function agentChat(task: WorkflowTask): Promise { } = getActivities() let placeholderCommentId: string | undefined + let agentWorkerQueue = '' try { + // 0. Discover queue + agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) + // Update status to processing - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) @@ -47,13 +51,13 @@ export async function agentChat(task: WorkflowTask): Promise { nonRetryable: true, }) } - const userComment = await executeActivity(TaskQueueAgent, getCommentActivity, userCommentId) + const userComment = await executeActivity(agentWorkerQueue, getCommentActivity, userCommentId) if (!userComment) { throw ApplicationFailure.create({ message: 'User comment not found', nonRetryable: true }) } // 2. Create Placeholder Comment - const placeholder = await executeActivity(TaskQueueAgent, createCommentActivity, { + const placeholder = await executeActivity(agentWorkerQueue, createCommentActivity, { assetId: task.assetId, message: '__CHAT__', sessionId: sessionId || 'pending', @@ -63,7 +67,7 @@ export async function agentChat(task: WorkflowTask): Promise { placeholderCommentId = placeholder.id // 3. Get Asset - const asset = await executeActivity(TaskQueueAgent, getAssetActivity, task.assetId) + const asset = await executeActivity(agentWorkerQueue, getAssetActivity, task.assetId) if (!asset || !asset.project) { throw ApplicationFailure.create({ message: 'Asset or project not found', nonRetryable: true }) } @@ -71,7 +75,7 @@ export async function agentChat(task: WorkflowTask): Promise { // 4. Initialize Session if missing if (!sessionId) { - sessionId = await executeActivity(TaskQueueAgent, initializeAgentSessionActivity, { + sessionId = await executeActivity(agentWorkerQueue, initializeAgentSessionActivity, { teamId, agentId: agentId, userCommentId, @@ -80,7 +84,7 @@ export async function agentChat(task: WorkflowTask): Promise { } // 4b. Fetch Agent Context - const context = await executeActivity(TaskQueueAgent, getAgentChatContextActivity, { + const context = await executeActivity(agentWorkerQueue, getAgentChatContextActivity, { teamId, agentId: agentId, }) @@ -96,7 +100,7 @@ export async function agentChat(task: WorkflowTask): Promise { } const pathContext = await executeActivity( - TaskQueueAgent, + agentWorkerQueue, getAssetPathContextActivity, task.assetId, ) @@ -126,8 +130,6 @@ export async function agentChat(task: WorkflowTask): Promise { folderId = asset.parentId } - const agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) - const aiResult = await executeActivity(agentWorkerQueue, agentChatActivity, { teamId, agentId: agentId, @@ -145,9 +147,9 @@ export async function agentChat(task: WorkflowTask): Promise { // 7. Update Placeholder Comment if (placeholderCommentId) { if (aiResult.text.trim() === '__NO_REPLY__') { - await executeActivity(TaskQueueAgent, deleteCommentActivity, placeholderCommentId) + await executeActivity(agentWorkerQueue, deleteCommentActivity, placeholderCommentId) } else { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: aiResult.text, sessionId: aiResult.sessionId, @@ -157,7 +159,7 @@ export async function agentChat(task: WorkflowTask): Promise { // 8. Update Usage if (aiResult.usage) { - await executeActivity(TaskQueueAgent, updateTaskUsageActivity, { + await executeActivity(agentWorkerQueue, updateTaskUsageActivity, { taskId: task.id, inputTokens: aiResult.usage.inputTokens, outputTokens: aiResult.usage.outputTokens, @@ -166,7 +168,7 @@ export async function agentChat(task: WorkflowTask): Promise { } // Update status to completed - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'completed', output: { sessionId: aiResult.sessionId }, @@ -175,14 +177,14 @@ export async function agentChat(task: WorkflowTask): Promise { console.error(`AgentChat failed for task ${task.id}:`, err) // Update placeholder comment with error message - if (placeholderCommentId) { + if (placeholderCommentId && agentWorkerQueue) { try { const errorMessage = err instanceof Error && err.message.startsWith('AI error:') ? `AI error: ${err.message.substring(9)}` : 'Sorry, I encountered an error while processing your request.' - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: errorMessage, }) @@ -192,11 +194,13 @@ export async function agentChat(task: WorkflowTask): Promise { } // Update status to failed - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { - taskId: task.id, - status: 'failed', - output: { error: err instanceof Error ? err.message : String(err) }, - }) + if (agentWorkerQueue) { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { + taskId: task.id, + status: 'failed', + output: { error: err instanceof Error ? err.message : String(err) }, + }) + } throw err } } diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index 19acc5af..da6b812f 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -15,17 +15,21 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } = getActivities() let placeholderCommentId: string | undefined + let agentWorkerQueue = '' try { + // 0. Discover queue + agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) + // Update status to processing - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) // 0. Create Placeholder Comment const payload = task.payload - const placeholder = await executeActivity(TaskQueueAgent, createCommentActivity, { + const placeholder = await executeActivity(agentWorkerQueue, createCommentActivity, { assetId: task.assetId, message: '__EMBEDDING__', sessionId: payload?.agent?.sessionId || task.id, @@ -38,13 +42,11 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } // 1. Fetch Agent Context - const context = await executeActivity(TaskQueueAgent, getEmbeddingContextActivity, { + const context = await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, { teamId: task.teamId, assetId: task.assetId, }) - const agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) - // Call the activity to generate embeddings const result = await executeActivity(agentWorkerQueue, generateEmbeddingActivity, { teamId: task.teamId, @@ -54,7 +56,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // Save computed embeddings if (result.embeddings.length > 0) { - await executeActivity(TaskQueueAgent, saveAssetEmbeddingsActivity, { + await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, { assetId: task.assetId, embeddings: result.embeddings, }) @@ -62,7 +64,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // Update Usage if (result.usage) { - await executeActivity(TaskQueueAgent, updateTaskUsageActivity, { + await executeActivity(agentWorkerQueue, updateTaskUsageActivity, { taskId: task.id, inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens, @@ -72,14 +74,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // 7. Update Placeholder Comment if (placeholderCommentId) { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: 'Embedding completed successfully.', }) } // Update status to completed - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -87,9 +89,9 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { console.error(`AgentEmbeddingMedia failed for task ${task.id}:`, err) // Update placeholder comment with error message - if (placeholderCommentId) { + if (placeholderCommentId && agentWorkerQueue) { try { - await executeActivity(TaskQueueAgent, updateCommentActivity, { + await executeActivity(agentWorkerQueue, updateCommentActivity, { commentId: placeholderCommentId, message: `Embedding failed: ${err instanceof Error ? err.message : String(err)}`, }) @@ -99,11 +101,13 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } // Update status to failed - await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { - taskId: task.id, - status: 'failed', - output: { error: err instanceof Error ? err.message : String(err) }, - }) + if (agentWorkerQueue) { + await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { + taskId: task.id, + status: 'failed', + output: { error: err instanceof Error ? err.message : String(err) }, + }) + } throw err } } diff --git a/packages/transcode/src/index.ts b/packages/transcode/src/index.ts index 44ef590e..a9ee849e 100644 --- a/packages/transcode/src/index.ts +++ b/packages/transcode/src/index.ts @@ -7,3 +7,8 @@ export function initTranscodeWorkflows() { registerWorkflow(WorkflowTaskType.transcode, transcodeMedia) registerActivities(transcodeActivities) } + +export * from './transcoder' +export * from './transcode' +export * from './workflows/transcode' +export * from './activities/transcode' diff --git a/packages/transcode/src/workflows/transcode.ts b/packages/transcode/src/workflows/transcode.ts index b720fb67..c5d8bda4 100644 --- a/packages/transcode/src/workflows/transcode.ts +++ b/packages/transcode/src/workflows/transcode.ts @@ -1,8 +1,8 @@ import type { WorkflowTask } from '@shumai/db' import { - executeActivity, - getActivities, - TaskQueueTranscode, + executeActivity, + getActivities, + TaskQueueTranscode, } from '@shumai/workflow-core/src/workflow-utils' import { ApplicationFailure } from '@temporalio/workflow' @@ -25,19 +25,22 @@ export async function transcodeMedia(task: WorkflowTask): Promise { let workerQueue = '' try { + // 0. Get Worker-Specific Queue for Transcode + workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity) + // 1. Update status to processing - await executeActivity(TaskQueueTranscode, updateTaskStatusActivity, { + await executeActivity(workerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) - await executeActivity(TaskQueueTranscode, updateAssetStatusActivity, { + await executeActivity(workerQueue, updateAssetStatusActivity, { assetId: task.assetId, status: 'processing', }) // 2. Get Asset - const asset = await executeActivity(TaskQueueTranscode, getAssetActivity, task.assetId) + const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId) const key = asset?.storageKey?.key if (!asset || !key) { throw ApplicationFailure.create({ @@ -46,9 +49,6 @@ export async function transcodeMedia(task: WorkflowTask): Promise { }) } - // 3. Get Worker-Specific Queue for Transcode - workerQueue = await executeActivity(TaskQueueTranscode, getTranscodeWorkerQueueActivity) - // 4. Download Media to Tmp const download = await executeActivity(workerQueue, downloadMediaToTmpActivity, { assetKey: key, @@ -173,29 +173,31 @@ export async function transcodeMedia(task: WorkflowTask): Promise { } // 11. Update Asset Media and Status - await executeActivity(TaskQueueTranscode, updateAssetMediaActivity, { + await executeActivity(workerQueue, updateAssetMediaActivity, { assetId: asset.id, mediaInfo, }) - await executeActivity(TaskQueueTranscode, updateAssetStatusActivity, { + await executeActivity(workerQueue, updateAssetStatusActivity, { assetId: asset.id, status: 'processed', }) // 12. Update Task Status - await executeActivity(TaskQueueTranscode, updateTaskStatusActivity, { + await executeActivity(workerQueue, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) } catch (err) { console.error(`TranscodeMedia failed for task ${task.id}:`, err) // Update status to failed - await executeActivity(TaskQueueTranscode, updateTaskStatusActivity, { - taskId: task.id, - status: 'failed', - output: { error: err instanceof Error ? err.message : String(err) }, - }) + if (workerQueue) { + await executeActivity(workerQueue, updateTaskStatusActivity, { + taskId: task.id, + status: 'failed', + output: { error: err instanceof Error ? err.message : String(err) }, + }) + } throw err } finally { if (tmpDir && workerQueue) { diff --git a/packages/workflow-core/src/workflow.ts b/packages/workflow-core/src/workflow.ts index 39ba41f7..1a64b3a8 100644 --- a/packages/workflow-core/src/workflow.ts +++ b/packages/workflow-core/src/workflow.ts @@ -1,5 +1,6 @@ import { prisma, registerWorkflowTrigger, WorkflowTask } from '@shumai/db' import path from 'path' +import * as taskActivities from './activities/task' import { Executor } from './executor' import { LocalExecutor } from './local-executor' import { TemporalExecutor } from './temporal-executor' @@ -117,8 +118,8 @@ export class WorkflowService { const specificWorker = await Worker.create({ connection, - // No workflows needed for the unique queue - activities, + // Register both domain-specific activities and shared task activities + activities: { ...activities, ...taskActivities }, taskQueue: workerSpecificQueue, bundlerOptions: { webpackConfigHook: (config) => {