diff --git a/packages/agent/src/activities/ai.test.ts b/packages/agent/src/activities/ai.test.ts index 3e110013..09e2e385 100644 --- a/packages/agent/src/activities/ai.test.ts +++ b/packages/agent/src/activities/ai.test.ts @@ -11,7 +11,6 @@ import { s3Service } from '@shumai/core/src/s3/s3' import { transcodeService } from '@shumai/core' import { prisma, AssetType, AssetStatus } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db/test' -import * as fs from 'fs' vi.mock('fs', async (importOriginal) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- importOriginal returns the raw fs module, typed dynamically as any here @@ -112,16 +111,14 @@ describe('AI Activities Unit Tests', () => { const res = await generateVideoChunkEmbeddingActivity({ teamId: 't1', - assetKey: 'test.mp4', - startTime: 0, - endTime: 60, + chunkKey: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4', }) expect(res.embedding).toEqual([0.1, 0.2, 0.3]) - expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'test.mp4') - expect(vi.mocked(fs.writeFileSync)).toHaveBeenCalled() - expect(vi.mocked(fs.readFileSync)).toHaveBeenCalled() - expect(vi.mocked(fs.unlinkSync)).toHaveBeenCalled() + expect(s3Service.getObject).toHaveBeenCalledWith( + 'shumai', + 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4', + ) }) it('should generate text embedding successfully', async () => { diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index c9cd78c6..35db05b2 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -310,7 +310,8 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse } const parsedDuration = parseFloat(process.env.EMBEDDING_CHUNK_DURATION || '') - const chunkDuration = Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0 + const chunkDuration = + Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0 return { agent, @@ -386,9 +387,7 @@ export async function generateImageEmbeddingActivity( export interface GenerateVideoChunkEmbeddingParams { teamId: string - assetKey: string - startTime: number - endTime: number + chunkKey: string } export async function generateVideoChunkEmbeddingActivity( @@ -414,44 +413,21 @@ export async function generateVideoChunkEmbeddingActivity( outputTokens: 0, } - const { buffer: data } = await s3Service.getObject( - process.env.S3_BUCKET || 'shumai', - params.assetKey, - ) - - const tmpFile = path.join(os.tmpdir(), `video-chunk-source-${Date.now()}.mp4`) - fs.writeFileSync(tmpFile, data) - - const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`) try { - await execFileAsync('ffmpeg', [ - '-y', - '-i', - tmpFile, - '-ss', - params.startTime.toString(), - '-t', - (params.endTime - params.startTime).toString(), - '-c', - 'copy', - chunkTmp, - ]) + const { buffer: data } = await s3Service.getObject( + process.env.S3_BUCKET || 'shumai', + params.chunkKey, + ) - const chunkData = fs.readFileSync(chunkTmp) - fs.unlinkSync(chunkTmp) - - const embVec = await generateMultimodalEmbedding(ai, chunkData, 'video/mp4') + const embVec = await generateMultimodalEmbedding(ai, data, 'video/mp4') return { embedding: embVec, usage, } } catch (err) { throw ApplicationFailure.create({ - message: `Failed to generate video chunk embedding for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`, + message: `Failed to generate video chunk embedding for key ${params.chunkKey}: ${err instanceof Error ? err.message : String(err)}`, nonRetryable: false, }) - } finally { - if (fs.existsSync(tmpFile)) fs.unlinkSync(tmpFile) - if (fs.existsSync(chunkTmp)) fs.unlinkSync(chunkTmp) } } diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts index 3764b641..43725acc 100644 --- a/packages/agent/src/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -44,9 +44,25 @@ describe('Agent Embedding Workflow', () => { getAgentWorkerQueueActivity: Object.assign(vi.fn(), { _activityName: 'getAgentWorkerQueueActivity', }), + getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), { + _activityName: 'getTranscodeWorkerQueueActivity', + }), + downloadMediaToTmpActivity: Object.assign(vi.fn(), { + _activityName: 'downloadMediaToTmpActivity', + }), + cleanupTmpDirActivity: Object.assign(vi.fn(), { + _activityName: 'cleanupTmpDirActivity', + }), + transcodeVideoChunkActivity: Object.assign(vi.fn(), { + _activityName: 'transcodeVideoChunkActivity', + }), + deleteS3ObjectActivity: Object.assign(vi.fn(), { + _activityName: 'deleteS3ObjectActivity', + }), } mockActivities.getAgentWorkerQueueActivity.mockResolvedValue('agent_queue') + mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_queue') mockActivities.updateTaskStatusActivity.mockResolvedValue({}) mockActivities.createCommentActivity.mockResolvedValue({ id: 'comment-placeholder-id' }) mockActivities.getEmbeddingContextActivity.mockResolvedValue({ @@ -66,6 +82,19 @@ describe('Agent Embedding Workflow', () => { embedding: [0.3, 0.4], usage: { inputTokens: 3, outputTokens: 3, model: 'gpt' }, }) + mockActivities.downloadMediaToTmpActivity.mockResolvedValue({ + filePath: '/tmp/test.mp4', + tmpDir: '/tmp/test-dir', + }) + mockActivities.transcodeVideoChunkActivity.mockImplementation( + async (params: { assetId: string; startTime: number; endTime: number }) => { + return { + chunkKey: `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}.mp4`, + } + }, + ) + mockActivities.deleteS3ObjectActivity.mockResolvedValue(undefined) + mockActivities.cleanupTmpDirActivity.mockResolvedValue(undefined) // eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities contains vi.fn mock functions which are cast to expected activity proxy types vi.mocked(workflowUtils.getActivities).mockReturnValue(mockActivities as any) @@ -156,8 +185,8 @@ describe('Agent Embedding Workflow', () => { mediaType: 'image/png', storageKey: { key: 'test.png' }, media: { - imageTranscodes: [{ key: 'test-transcoded.webp', isRaw: false, format: 'webp' }] - } + imageTranscodes: [{ key: 'test-transcoded.webp', isRaw: false, format: 'webp' }], + }, }, chunkDuration: 60.0, }) @@ -210,27 +239,59 @@ describe('Agent Embedding Workflow', () => { await agentEmbeddingMedia(task) - // Verify video chunk embedding generation (3 chunks: 0-60, 60-120, 120-150) - expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenCalledTimes(3) - expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(1, { - teamId: 't1', + // Verify video transcode queue discovery & download + expect(mockActivities.getTranscodeWorkerQueueActivity).toHaveBeenCalled() + expect(mockActivities.downloadMediaToTmpActivity).toHaveBeenCalledWith({ assetKey: 'test-transcoded.mp4', + }) + + // Verify video chunk slicing activity called 3 times on transcode queue + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(3) + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(1, { + assetId: 'a1', + filePath: '/tmp/test.mp4', startTime: 0, endTime: 60, }) - expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, { - teamId: 't1', - assetKey: 'test-transcoded.mp4', + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, { + assetId: 'a1', + filePath: '/tmp/test.mp4', startTime: 60, endTime: 120, }) - expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, { - teamId: 't1', - assetKey: 'test-transcoded.mp4', + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, { + assetId: 'a1', + filePath: '/tmp/test.mp4', startTime: 120, endTime: 150, }) + // Verify video chunk embedding generation (3 chunks) on agent queue + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenCalledTimes(3) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(1, { + teamId: 't1', + chunkKey: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4', + }) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, { + teamId: 't1', + chunkKey: 'files/a1/tmp-embedding-chunks/chunk-60-120.mp4', + }) + expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, { + teamId: 't1', + chunkKey: 'files/a1/tmp-embedding-chunks/chunk-120-150.mp4', + }) + + // Verify deletion of temporary chunks from S3 + expect(mockActivities.deleteS3ObjectActivity).toHaveBeenCalledTimes(3) + expect(mockActivities.deleteS3ObjectActivity).toHaveBeenNthCalledWith(1, { + key: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4', + }) + + // Verify temp directory cleanup on transcode queue + expect(mockActivities.cleanupTmpDirActivity).toHaveBeenCalledWith({ + tmpDir: '/tmp/test-dir', + }) + // Verify usage update (3 chunks * 3 tokens = 9) expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith({ taskId: task.id, diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index d0f063b4..1c829455 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -1,6 +1,11 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core' +import { + getActivities, + executeActivity, + TaskQueueAgent, + TaskQueueTranscode, +} from '@shumai/workflow-core' export interface GeneratedEmbedding { embedding: number[] @@ -19,14 +24,25 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { createCommentActivity, updateCommentActivity, getAgentWorkerQueueActivity, + getTranscodeWorkerQueueActivity, + downloadMediaToTmpActivity, + cleanupTmpDirActivity, + transcodeVideoChunkActivity, + deleteS3ObjectActivity, } = getActivities() let placeholderCommentId: string | undefined let agentWorkerQueue = '' + let transcodeWorkerQueue = '' + let tmpDir: string | undefined try { - // 0. Discover queue + // 0. Discover queues agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity) + transcodeWorkerQueue = await executeActivity( + TaskQueueTranscode, + getTranscodeWorkerQueueActivity, + ) // Update status to processing await executeActivity(agentWorkerQueue, updateTaskStatusActivity, { @@ -77,7 +93,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { if (media.videoTranscodes && media.videoTranscodes.length > 0) { // Find first non-raw transcode or fallback to first transcode const transcode = - media.videoTranscodes.find((t: PrismaJson.VideoTranscode) => !t.isRaw) || media.videoTranscodes[0] + media.videoTranscodes.find((t: PrismaJson.VideoTranscode) => !t.isRaw) || + media.videoTranscodes[0] if (transcode?.key) { embeddingKey = transcode.key } @@ -86,7 +103,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { if (media.imageTranscodes && media.imageTranscodes.length > 0) { // Find first non-raw transcode or fallback to first transcode const transcode = - media.imageTranscodes.find((t: PrismaJson.ImageTranscode) => !t.isRaw) || media.imageTranscodes[0] + media.imageTranscodes.find((t: PrismaJson.ImageTranscode) => !t.isRaw) || + media.imageTranscodes[0] if (transcode?.key) { embeddingKey = transcode.key if (transcode.format) { @@ -120,6 +138,13 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { usage.outputTokens += result.usage.outputTokens || 0 } } else if (isVideo) { + // 1. Download full video to transcode worker temp space + const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, { + assetKey: embeddingKey, + }) + const { filePath } = download + tmpDir = download.tmpDir + // Get video duration // eslint-disable-next-line @typescript-eslint/no-explicit-any const duration = (asset.media as any)?.duration || 0 @@ -129,26 +154,41 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { let end = start + limit if (end > duration) end = duration - const chunkResult = await executeActivity( - agentWorkerQueue, - generateVideoChunkEmbeddingActivity, - { - teamId: task.teamId, - assetKey: embeddingKey, - startTime: start, - endTime: end, - }, - ) - - results.push({ - embedding: chunkResult.embedding, + // Step A: Slice chunk on transcode worker and upload to S3 + const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, { + assetId: asset.id, + filePath, startTime: start, endTime: end, }) + const { chunkKey } = chunkRes - if (chunkResult.usage) { - usage.inputTokens += chunkResult.usage.inputTokens || 0 - usage.outputTokens += chunkResult.usage.outputTokens || 0 + try { + // Step B: Generate embedding on agent worker + const chunkResult = await executeActivity( + agentWorkerQueue, + generateVideoChunkEmbeddingActivity, + { + teamId: task.teamId, + chunkKey, + }, + ) + + results.push({ + embedding: chunkResult.embedding, + startTime: start, + endTime: end, + }) + + if (chunkResult.usage) { + usage.inputTokens += chunkResult.usage.inputTokens || 0 + usage.outputTokens += chunkResult.usage.outputTokens || 0 + } + } finally { + // Step C: Delete temporary chunk from S3 + await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, { + key: chunkKey, + }) } } } @@ -206,6 +246,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { }) } throw err + } finally { + if (tmpDir && transcodeWorkerQueue) { + try { + await executeActivity(transcodeWorkerQueue, cleanupTmpDirActivity, { tmpDir }) + } catch (cleanupErr) { + console.error('Failed to cleanup transcode worker tmp dir:', cleanupErr) + } + } } } diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 83cedfb5..f0ba24e8 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -4,6 +4,11 @@ import { prisma } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db/test' import { beforeEach, describe, expect, it, vi } from 'vitest' import { transcodeService } from '@shumai/core' +import * as child_process from 'child_process' + +vi.mock('child_process', () => ({ + execFile: vi.fn(), +})) import { getMediaInfoActivity, overlayAnnotationsActivity, @@ -13,6 +18,8 @@ import { downloadMediaToTmpActivity, transcodeImageActivity, generateSpriteActivity, + transcodeVideoChunkActivity, + deleteS3ObjectActivity, } from './transcode' vi.mock('@shumai/core/src/s3/s3', () => ({ @@ -22,6 +29,7 @@ vi.mock('@shumai/core/src/s3/s3', () => ({ headObject: vi.fn(), presign: vi.fn(), downloadToFile: vi.fn(), + deleteObject: vi.fn(), }, })) @@ -365,4 +373,56 @@ describe('Transcode Activities', () => { ).rejects.toThrowError(/Overlay annotations failed/) }) }) + + describe('Video Chunk Activities', () => { + it('should transcode video chunk using ffmpeg and upload to S3', async () => { + // Mock child_process.execFile to simulate ffmpeg succeeding + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(child_process.execFile as any).mockImplementation( + (file: string, args: string[], cb: any) => { + cb(null, { stdout: '', stderr: '' }) + }, + ) + + const res = await transcodeVideoChunkActivity({ + assetId: 'asset-abc', + filePath: '/tmp/full-video.mp4', + startTime: 10, + endTime: 25, + }) + + // Verify ffmpeg command args + expect(child_process.execFile).toHaveBeenCalledWith( + 'ffmpeg', + expect.arrayContaining([ + '-y', + '-i', + '/tmp/full-video.mp4', + '-ss', + '10', + '-t', + '15', + '-c', + 'copy', + ]), + expect.any(Function), + ) + + // Verify S3 upload + expect(s3Service.putObject).toHaveBeenCalledWith( + 'shumai', + expect.stringMatching(/^files\/asset-abc\/tmp-embedding-chunks\/chunk-10-25-.*\.mp4$/), + expect.any(Buffer), + expect.any(Number), + 'video/mp4', + ) + + expect(res.chunkKey).toMatch(/^files\/asset-abc\/tmp-embedding-chunks\/chunk-10-25-.*\.mp4$/) + }) + + it('should delete S3 object successfully', async () => { + await deleteS3ObjectActivity({ key: 'files/test-key.mp4' }) + expect(s3Service.deleteObject).toHaveBeenCalledWith('shumai', 'files/test-key.mp4') + }) + }) }) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index 630484cc..c3722a6c 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -5,6 +5,12 @@ import { metadataService } from '@shumai/core/src/metadata/metadata' import { ApplicationFailure } from '@temporalio/activity' import * as path from 'path' import * as fs from 'fs' +import * as os from 'os' +import { execFile } from 'child_process' +import { promisify } from 'util' +import { ulid } from 'ulid' + +const execFileAsync = promisify(execFile) export interface GetMediaInfoActivityParams { assetId: string @@ -539,3 +545,59 @@ export async function createEmbeddingTaskIfEnabledActivity( }, }) } + +export interface TranscodeVideoChunkParams { + assetId: string + filePath: string + startTime: number + endTime: number +} + +export async function transcodeVideoChunkActivity( + params: TranscodeVideoChunkParams, +): Promise<{ chunkKey: string }> { + const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`) + try { + // Slice video segment using ffmpeg + await execFileAsync('ffmpeg', [ + '-y', + '-i', + params.filePath, + '-ss', + params.startTime.toString(), + '-t', + (params.endTime - params.startTime).toString(), + '-c', + 'copy', + chunkTmp, + ]) + + const chunkData = fs.readFileSync(chunkTmp) + + // Upload to S3 + const bucket = process.env.S3_BUCKET || 'shumai' + const key = `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}-${ulid()}.mp4` + + await s3Service.putObject(bucket, key, chunkData, chunkData.length, 'video/mp4') + + return { chunkKey: key } + } catch (err) { + throw ApplicationFailure.create({ + message: `Failed to transcode video chunk for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`, + nonRetryable: false, + }) + } finally { + if (fs.existsSync(chunkTmp)) { + try { + fs.unlinkSync(chunkTmp) + } catch (e) { + console.error('Failed to cleanup local chunk file:', e) + } + } + } +} + +export async function deleteS3ObjectActivity(params: { key: string }): Promise { + const bucket = process.env.S3_BUCKET || 'shumai' + await s3Service.deleteObject(bucket, params.key) +}