From f399d2c469bebad0724a2d3583d7cd4196f58c72 Mon Sep 17 00:00:00 2001 From: yiling Date: Thu, 18 Jun 2026 10:57:22 +0800 Subject: [PATCH] feat(embedding): defer embedding workflow, use transcoded assets, and execute chunks in separate activities --- docs/configuration/env-variables.mdx | 9 +- docs/using-shumai/agents.mdx | 1 + packages/agent/src/activities/ai.test.ts | 36 ++--- packages/agent/src/activities/ai.ts | 116 ++++++++++++++++ packages/agent/src/index.ts | 2 + .../src/workflows/agent-embedding.test.ts | 130 +++++++++++++++--- .../agent/src/workflows/agent-embedding.ts | 129 ++++++++++++++--- packages/core/src/upload/upload.ts | 27 ---- .../transcode/src/activities/transcode.ts | 73 +++++++++- .../transcode/src/workflows/transcode.test.ts | 15 ++ packages/transcode/src/workflows/transcode.ts | 8 ++ 11 files changed, 451 insertions(+), 95 deletions(-) diff --git a/docs/configuration/env-variables.mdx b/docs/configuration/env-variables.mdx index 2ee3aee4..fbc9bd88 100644 --- a/docs/configuration/env-variables.mdx +++ b/docs/configuration/env-variables.mdx @@ -54,10 +54,11 @@ Shumai supports storing files locally or on S3-compatible endpoints. Configures whether background transcoding and AI agents are executed locally or offloaded to a Temporal cluster. -| Variable | Description | Example / Default | -| :------------------ | :------------------------------------------------------------------------------------------------- | :------------------------------ | -| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` | -| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) | +| Variable | Description | Example / Default | +| :------------------------- | :------------------------------------------------------------------------------------------------- | :------------------------------ | +| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` | +| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) | +| `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) | --- diff --git a/docs/using-shumai/agents.mdx b/docs/using-shumai/agents.mdx index d4eca198..019f0713 100644 --- a/docs/using-shumai/agents.mdx +++ b/docs/using-shumai/agents.mdx @@ -34,6 +34,7 @@ A specialized utility agent that processes assets to generate search indexes. Yo - Once created and enabled, Shumai automatically runs embedding generation for new file uploads if the file type is supported (image or video). - These search indexes are saved to power Shumai's natural language semantic search capabilities. - **Note:** The Embedding Agent is hardcoded to use `google-embedding-2` as the LLM model. Because of this, you cannot choose a provider or model. Make sure you set the `GEMINI_API_KEY` environment variable to make it work. +- **Chunk Duration:** You can configure the duration of chunks used for long video indexing via the `EMBEDDING_CHUNK_DURATION` environment variable (defaults to 60 seconds). --- diff --git a/packages/agent/src/activities/ai.test.ts b/packages/agent/src/activities/ai.test.ts index 044c7223..5807f7ae 100644 --- a/packages/agent/src/activities/ai.test.ts +++ b/packages/agent/src/activities/ai.test.ts @@ -1,10 +1,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { - generateEmbeddingActivity, generateTextEmbeddingActivity, extractAiMetadataActivity, getEmbeddingContextActivity, saveAssetEmbeddingsActivity, + generateImageEmbeddingActivity, + generateVideoChunkEmbeddingActivity, } from './ai' import { s3Service } from '@shumai/core/src/s3/s3' import { transcodeService } from '@shumai/core' @@ -93,19 +94,13 @@ describe('AI Activities Unit Tests', () => { contentType: 'image/png', } as unknown as { buffer: Buffer; contentType: string }) - const context = { - agent: { config: { provider: 'google', model: 'gemini' } }, - asset: { id: 'a1', mediaType: 'image/png', storageKey: { key: 'test.png' } }, - } - - const res = await generateEmbeddingActivity({ + const res = await generateImageEmbeddingActivity({ teamId: 't1', - assetId: 'a1', - context, + assetKey: 'test.png', + mediaType: 'image/png', }) - expect(res.embeddings.length).toBe(1) - expect(res.embeddings[0].embedding).toEqual([0.1, 0.2, 0.3]) + expect(res.embedding).toEqual([0.1, 0.2, 0.3]) expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'test.png') }) @@ -115,22 +110,15 @@ describe('AI Activities Unit Tests', () => { contentType: 'video/mp4', } as unknown as { buffer: Buffer; contentType: string }) - const context = { - agent: { config: { provider: 'google', model: 'gemini' } }, - asset: { id: 'a2', mediaType: 'video/mp4', storageKey: { key: 'test.mp4' } }, - } - - const res = await generateEmbeddingActivity({ + const res = await generateVideoChunkEmbeddingActivity({ teamId: 't1', - assetId: 'a2', - context, + assetKey: 'test.mp4', + startTime: 0, + endTime: 60, }) - // Duration is 120s, chunks are 60s, so it should generate 2 chunks (0-60, 60-120) - expect(res.embeddings.length).toBe(2) - expect(res.embeddings[0]).toEqual({ embedding: [0.1, 0.2, 0.3], startTime: 0, endTime: 60 }) - expect(res.embeddings[1]).toEqual({ embedding: [0.1, 0.2, 0.3], startTime: 60, endTime: 120 }) - + expect(res.embedding).toEqual([0.1, 0.2, 0.3]) + expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'test.mp4') expect(vi.mocked(fs.writeFileSync)).toHaveBeenCalled() expect(vi.mocked(fs.readFileSync)).toHaveBeenCalled() expect(vi.mocked(fs.unlinkSync)).toHaveBeenCalled() diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index 41c96d67..afc95600 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -309,9 +309,12 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true }) } + const chunkDuration = parseFloat(process.env.EMBEDDING_CHUNK_DURATION || '60.0') || 60.0 + return { agent, asset, + chunkDuration, } } @@ -338,3 +341,116 @@ export async function saveAssetEmbeddingsActivity(params: { } } } + +export interface GenerateImageEmbeddingParams { + teamId: string + assetKey: string + mediaType: string +} + +export async function generateImageEmbeddingActivity( + params: GenerateImageEmbeddingParams, +): Promise<{ + embedding: number[] + usage: Usage +}> { + const apiKey = process.env.GEMINI_API_KEY + if (!apiKey) { + throw ApplicationFailure.create({ + message: 'GEMINI_API_KEY environment variable is not configured', + nonRetryable: true, + }) + } + + const resolvedApiKey = process.env[apiKey] || apiKey + const ai = new GoogleGenAI({ apiKey: resolvedApiKey }) + + const usage: Usage = { + model: 'gemini-embedding-2', + inputTokens: 0, + outputTokens: 0, + } + + const { buffer: data } = await s3Service.getObject( + process.env.S3_BUCKET || 'shumai', + params.assetKey, + ) + const embVec = await generateMultimodalEmbedding(ai, data, params.mediaType) + + return { + embedding: embVec, + usage, + } +} + +export interface GenerateVideoChunkEmbeddingParams { + teamId: string + assetKey: string + startTime: number + endTime: number +} + +export async function generateVideoChunkEmbeddingActivity( + params: GenerateVideoChunkEmbeddingParams, +): Promise<{ + embedding: number[] + usage: Usage +}> { + const apiKey = process.env.GEMINI_API_KEY + if (!apiKey) { + throw ApplicationFailure.create({ + message: 'GEMINI_API_KEY environment variable is not configured', + nonRetryable: true, + }) + } + + const resolvedApiKey = process.env[apiKey] || apiKey + const ai = new GoogleGenAI({ apiKey: resolvedApiKey }) + + const usage: Usage = { + model: 'gemini-embedding-2', + inputTokens: 0, + outputTokens: 0, + } + + const { buffer: data } = await s3Service.getObject( + process.env.S3_BUCKET || 'shumai', + params.assetKey, + ) + + const tmpFile = path.join(os.tmpdir(), `video-chunk-source-${Date.now()}.mp4`) + fs.writeFileSync(tmpFile, data) + + const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`) + try { + await execFileAsync('ffmpeg', [ + '-y', + '-i', + tmpFile, + '-ss', + params.startTime.toString(), + '-t', + (params.endTime - params.startTime).toString(), + '-c', + 'copy', + chunkTmp, + ]) + + const chunkData = fs.readFileSync(chunkTmp) + fs.unlinkSync(chunkTmp) + + const embVec = await generateMultimodalEmbedding(ai, chunkData, 'video/mp4') + return { + embedding: embVec, + usage, + } + } catch (err) { + throw ApplicationFailure.create({ + message: `Failed to generate video chunk embedding for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`, + nonRetryable: false, + }) + } finally { + if (fs.existsSync(tmpFile)) fs.unlinkSync(tmpFile) + if (fs.existsSync(chunkTmp)) fs.unlinkSync(chunkTmp) + } +} diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 86915aef..0e9f24bb 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -380,6 +380,8 @@ export { generateTextEmbeddingActivity, type ExtractAiMetadataParams, extractAiMetadataActivity, + generateImageEmbeddingActivity, + generateVideoChunkEmbeddingActivity, } from './activities/ai' export * from './workflows/agent-autofill' export * from './workflows/agent-chat' diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts index 57441d7d..4ad9c024 100644 --- a/packages/agent/src/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -29,8 +29,11 @@ describe('Agent Embedding Workflow', () => { getEmbeddingContextActivity: Object.assign(vi.fn(), { _activityName: 'getEmbeddingContextActivity', }), - generateEmbeddingActivity: Object.assign(vi.fn(), { - _activityName: 'generateEmbeddingActivity', + generateImageEmbeddingActivity: Object.assign(vi.fn(), { + _activityName: 'generateImageEmbeddingActivity', + }), + generateVideoChunkEmbeddingActivity: Object.assign(vi.fn(), { + _activityName: 'generateVideoChunkEmbeddingActivity', }), saveAssetEmbeddingsActivity: Object.assign(vi.fn(), { _activityName: 'saveAssetEmbeddingsActivity', @@ -48,13 +51,21 @@ describe('Agent Embedding Workflow', () => { mockActivities.createCommentActivity.mockResolvedValue({ id: 'comment-placeholder-id' }) mockActivities.getEmbeddingContextActivity.mockResolvedValue({ agent: { id: 'b1' }, - asset: { id: 'a1' }, - dbProvider: { name: 'google' }, + asset: { + id: 'a1', + mediaType: 'image/png', + storageKey: { key: 'test.png' }, + }, + chunkDuration: 60.0, }) - mockActivities.generateEmbeddingActivity.mockResolvedValue({ - embeddings: [{ embedding: [0.1, 0.2] }], + mockActivities.generateImageEmbeddingActivity.mockResolvedValue({ + embedding: [0.1, 0.2], usage: { inputTokens: 5, outputTokens: 5, model: 'gpt' }, }) + mockActivities.generateVideoChunkEmbeddingActivity.mockResolvedValue({ + embedding: [0.3, 0.4], + usage: { inputTokens: 3, outputTokens: 3, model: 'gpt' }, + }) // eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities contains vi.fn mock functions which are cast to expected activity proxy types vi.mocked(workflowUtils.getActivities).mockReturnValue(mockActivities as any) @@ -64,7 +75,7 @@ describe('Agent Embedding Workflow', () => { }) }) - it('should run agent embedding workflow successfully', async () => { + it('should run agent embedding workflow successfully for image', async () => { const task = await prisma.workflowTask.create({ data: { type: 'ai_embedding', @@ -103,15 +114,11 @@ describe('Agent Embedding Workflow', () => { assetId: 'a1', }) - // Verify embedding generation - expect(mockActivities.generateEmbeddingActivity).toHaveBeenCalledWith({ + // Verify image embedding generation + expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({ teamId: 't1', - assetId: 'a1', - context: { - agent: { id: 'b1' }, - asset: { id: 'a1' }, - dbProvider: { name: 'google' }, - }, + assetKey: 'test.png', + mediaType: 'image/png', }) // Verify embeddings saved @@ -125,7 +132,7 @@ describe('Agent Embedding Workflow', () => { taskId: task.id, inputTokens: 5, outputTokens: 5, - model: 'gpt', + model: 'gemini-embedding-2', }) // Verify placeholder comment updated @@ -141,6 +148,76 @@ describe('Agent Embedding Workflow', () => { }) }) + it('should run agent embedding workflow successfully for video in chunks', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'a1', + mediaType: 'video/mp4', + storageKey: { key: 'test.mp4' }, + media: { + duration: 150.0, + videoTranscodes: [{ key: 'test-transcoded.mp4', isRaw: false }], + }, + }, + chunkDuration: 60.0, + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'a1', + teamId: 't1', + payload: { + projectId: 'p1', + agent: { sessionId: 's1', agentId: 'agent-1' }, + }, + }, + }) + + await agentEmbeddingMedia(task) + + // Verify video chunk embedding generation (3 chunks: 0-60, 60-120, 120-150) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenCalledTimes(3) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(1, { + teamId: 't1', + assetKey: 'test-transcoded.mp4', + startTime: 0, + endTime: 60, + }) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, { + teamId: 't1', + assetKey: 'test-transcoded.mp4', + startTime: 60, + endTime: 120, + }) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, { + teamId: 't1', + assetKey: 'test-transcoded.mp4', + startTime: 120, + endTime: 150, + }) + + // Verify usage update (3 chunks * 3 tokens = 9) + expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith({ + taskId: task.id, + inputTokens: 9, + outputTokens: 9, + model: 'gemini-embedding-2', + }) + + // Verify embeddings saved + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({ + assetId: 'a1', + embeddings: [ + { embedding: [0.3, 0.4], startTime: 0, endTime: 60 }, + { embedding: [0.3, 0.4], startTime: 60, endTime: 120 }, + { embedding: [0.3, 0.4], startTime: 120, endTime: 150 }, + ], + }) + }) + it('should throw if task has no teamId', async () => { const task = await prisma.workflowTask.create({ data: { @@ -160,10 +237,18 @@ describe('Agent Embedding Workflow', () => { }) }) - it('should skip saving embeddings if none are returned', async () => { - mockActivities.generateEmbeddingActivity.mockResolvedValue({ - embeddings: [], - usage: null, + it('should skip saving embeddings if none are returned (video duration 0)', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'a1', + mediaType: 'video/mp4', + storageKey: { key: 'test.mp4' }, + media: { + duration: 0.0, + }, + }, + chunkDuration: 60.0, }) const task = await prisma.workflowTask.create({ @@ -178,7 +263,6 @@ describe('Agent Embedding Workflow', () => { await agentEmbeddingMedia(task) expect(mockActivities.saveAssetEmbeddingsActivity).not.toHaveBeenCalled() - expect(mockActivities.updateTaskUsageActivity).not.toHaveBeenCalled() expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({ taskId: task.id, status: 'completed', @@ -186,7 +270,9 @@ describe('Agent Embedding Workflow', () => { }) it('should handle failures, update placeholder with error, and set status to failed', async () => { - mockActivities.generateEmbeddingActivity.mockRejectedValue(new Error('AI Service Unavailable')) + mockActivities.generateImageEmbeddingActivity.mockRejectedValue( + new Error('AI Service Unavailable'), + ) const task = await prisma.workflowTask.create({ data: { diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index da6b812f..b43fe1d8 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -2,11 +2,18 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core' +export interface GeneratedEmbedding { + embedding: number[] + startTime?: number + endTime?: number +} + export async function agentEmbeddingMedia(task: WorkflowTask): Promise { const { updateTaskStatusActivity, getEmbeddingContextActivity, - generateEmbeddingActivity, + generateImageEmbeddingActivity, + generateVideoChunkEmbeddingActivity, saveAssetEmbeddingsActivity, updateTaskUsageActivity, createCommentActivity, @@ -47,30 +54,118 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { assetId: task.assetId, }) - // Call the activity to generate embeddings - const result = await executeActivity(agentWorkerQueue, generateEmbeddingActivity, { - teamId: task.teamId, - assetId: task.assetId, - context, - }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { asset, chunkDuration } = context as any + + const isImage = asset.mediaType?.startsWith('image/') + const isVideo = asset.mediaType?.startsWith('video/') + + if (!isImage && !isVideo) { + throw ApplicationFailure.create({ + message: `unsupported media type for embeddings: ${asset.mediaType}`, + nonRetryable: true, + }) + } + + // Resolve the S3 key to use for embedding (preferring transcoded version) + let embeddingKey = asset.storageKey?.key + if (asset.media) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const media = asset.media as any + if (isVideo) { + if (media.videoTranscodes && media.videoTranscodes.length > 0) { + // Find first non-raw transcode or fallback to first transcode + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const transcode = + media.videoTranscodes.find((t: any) => !t.isRaw) || media.videoTranscodes[0] + if (transcode?.key) { + embeddingKey = transcode.key + } + } + } else if (isImage) { + if (media.imageTranscodes && media.imageTranscodes.length > 0) { + // Find first non-raw transcode or fallback to first transcode + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const transcode = + media.imageTranscodes.find((t: any) => !t.isRaw) || media.imageTranscodes[0] + if (transcode?.key) { + embeddingKey = transcode.key + } + } + } + } + + if (!embeddingKey) { + throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true }) + } + + const results: GeneratedEmbedding[] = [] + const usage = { + model: 'gemini-embedding-2', + inputTokens: 0, + outputTokens: 0, + } + + if (isImage) { + const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, { + teamId: task.teamId, + assetKey: embeddingKey, + mediaType: asset.mediaType, + }) + results.push({ embedding: result.embedding }) + if (result.usage) { + usage.inputTokens += result.usage.inputTokens || 0 + usage.outputTokens += result.usage.outputTokens || 0 + } + } else if (isVideo) { + // Get video duration + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const duration = (asset.media as any)?.duration || 0 + const limit = chunkDuration || 60.0 + + for (let start = 0.0; start < duration; start += limit) { + let end = start + limit + if (end > duration) end = duration + + const chunkResult = await executeActivity( + agentWorkerQueue, + generateVideoChunkEmbeddingActivity, + { + teamId: task.teamId, + assetKey: embeddingKey, + startTime: start, + endTime: end, + }, + ) + + results.push({ + embedding: chunkResult.embedding, + startTime: start, + endTime: end, + }) + + if (chunkResult.usage) { + usage.inputTokens += chunkResult.usage.inputTokens || 0 + usage.outputTokens += chunkResult.usage.outputTokens || 0 + } + } + } // Save computed embeddings - if (result.embeddings.length > 0) { + if (results.length > 0) { await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, { assetId: task.assetId, - embeddings: result.embeddings, + embeddings: results, }) } // Update Usage - if (result.usage) { - await executeActivity(agentWorkerQueue, updateTaskUsageActivity, { - taskId: task.id, - inputTokens: result.usage.inputTokens, - outputTokens: result.usage.outputTokens, - model: result.usage.model, - }) - } + await executeActivity(agentWorkerQueue, updateTaskUsageActivity, { + taskId: task.id, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + model: usage.model, + }) // 7. Update Placeholder Comment if (placeholderCommentId) { diff --git a/packages/core/src/upload/upload.ts b/packages/core/src/upload/upload.ts index bbcc08ff..cfab781d 100644 --- a/packages/core/src/upload/upload.ts +++ b/packages/core/src/upload/upload.ts @@ -294,33 +294,6 @@ export class UploadService { }) } - // Ai Embedding if enabled via agent - const embeddingAgent = await tx.agent.findFirst({ - where: { - type: 'embedding', - enabled: true, - user: { teamMembers: { some: { teamId: team.id } } }, - }, - }) - - if (embeddingAgent) { - if (asset.mediaType?.startsWith('video/') || asset.mediaType?.startsWith('image/')) { - await tx.workflowTask.create({ - data: { - assetId: asset.id, - type: WorkflowTaskType.ai_embedding, - status: WorkflowTaskStatus.pending, - teamId: team.id, - projectId, - payload: { - projectId, - agent: { agentId: embeddingAgent.id }, - }, - }, - }) - } - } - const isVideo = asset.mediaType?.startsWith('video/') const isImage = asset.mediaType?.startsWith('image/') if (!isVideo && !isImage) { diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index 5c16aaf0..630484cc 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -1,4 +1,4 @@ -import { prisma } from '@shumai/db' +import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db' import { s3Service } from '@shumai/core/src/s3/s3' import { transcodeService } from '@shumai/core' import { metadataService } from '@shumai/core/src/metadata/metadata' @@ -468,3 +468,74 @@ export async function overlayAnnotationsActivity(params: { throw err } } + +export interface CreateEmbeddingTaskIfEnabledParams { + assetId: string + teamId: string | null + projectId: string | null +} + +export async function createEmbeddingTaskIfEnabledActivity( + params: CreateEmbeddingTaskIfEnabledParams, +): Promise { + let teamId = params.teamId + let projectId = params.projectId + + // Resolve teamId and projectId from asset if missing + if (!teamId || !projectId) { + const asset = await prisma.asset.findUnique({ + where: { id: params.assetId }, + include: { + project: true, + }, + }) + if (asset) { + projectId = projectId || asset.projectId + teamId = teamId || asset.project?.teamId || null + } + } + + if (!teamId) { + return + } + + // Check if there is an active embedding agent for the team + const embeddingAgent = await prisma.agent.findFirst({ + where: { + type: 'embedding', + enabled: true, + user: { teamMembers: { some: { teamId } } }, + }, + }) + + if (!embeddingAgent) { + return + } + + // Check if a pending/processing embedding task already exists for this asset to avoid duplicates + const existing = await prisma.workflowTask.findFirst({ + where: { + assetId: params.assetId, + type: WorkflowTaskType.ai_embedding, + status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] }, + }, + }) + if (existing) { + return + } + + // Create embedding task + await prisma.workflowTask.create({ + data: { + assetId: params.assetId, + type: WorkflowTaskType.ai_embedding, + status: WorkflowTaskStatus.pending, + teamId, + projectId, + payload: { + projectId: projectId ?? '', + agent: { agentId: embeddingAgent.id }, + }, + }, + }) +} diff --git a/packages/transcode/src/workflows/transcode.test.ts b/packages/transcode/src/workflows/transcode.test.ts index 6a86490d..ecdec83d 100644 --- a/packages/transcode/src/workflows/transcode.test.ts +++ b/packages/transcode/src/workflows/transcode.test.ts @@ -41,6 +41,9 @@ describe('Transcode Workflow', () => { overlayAnnotationsActivity: Object.assign(vi.fn(), { _activityName: 'overlayAnnotationsActivity', }), + createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), { + _activityName: 'createEmbeddingTaskIfEnabledActivity', + }), } beforeEach(() => { @@ -134,6 +137,12 @@ describe('Transcode Workflow', () => { status: AssetStatus.processed, }) + expect(mockActivities.createEmbeddingTaskIfEnabledActivity).toHaveBeenCalledWith({ + assetId: 'asset-1', + teamId: 'team-1', + projectId: 'proj-1', + }) + expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({ taskId: 'task-1', status: WorkflowTaskStatus.completed, @@ -205,6 +214,12 @@ describe('Transcode Workflow', () => { assetId: 'asset-image', status: AssetStatus.processed, }) + + expect(mockActivities.createEmbeddingTaskIfEnabledActivity).toHaveBeenCalledWith({ + assetId: 'asset-image', + teamId: 'team-1', + projectId: 'proj-1', + }) }) it('should handle failures and update task status with error', async () => { diff --git a/packages/transcode/src/workflows/transcode.ts b/packages/transcode/src/workflows/transcode.ts index 522afd0b..d4944379 100644 --- a/packages/transcode/src/workflows/transcode.ts +++ b/packages/transcode/src/workflows/transcode.ts @@ -18,6 +18,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise { cleanupTmpDirActivity, takeScreenshotsActivity, overlayAnnotationsActivity, + createEmbeddingTaskIfEnabledActivity, } = getActivities() let tmpDir: string | undefined @@ -245,6 +246,13 @@ export async function transcodeMedia(task: WorkflowTask): Promise { status: 'processed', }) + // 11.5 Create AI embedding task if enabled + await executeActivity(workerQueue, createEmbeddingTaskIfEnabledActivity, { + assetId: asset.id, + teamId: task.teamId, + projectId: task.projectId, + }) + // 12. Update Task Status await executeActivity(workerQueue, updateTaskStatusActivity, { taskId: task.id,