feat(embedding): decouple video chunk transcode and embedding generation

This commit is contained in:
yiling
2026-06-18 11:54:30 +08:00
parent 9d1e8378f7
commit 46815736e2
6 changed files with 277 additions and 73 deletions
+5 -8
View File
@@ -11,7 +11,6 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
import { prisma, AssetType, AssetStatus } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import * as fs from 'fs'
vi.mock('fs', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- importOriginal returns the raw fs module, typed dynamically as any here
@@ -112,16 +111,14 @@ describe('AI Activities Unit Tests', () => {
const res = await generateVideoChunkEmbeddingActivity({
teamId: 't1',
assetKey: 'test.mp4',
startTime: 0,
endTime: 60,
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
})
expect(res.embedding).toEqual([0.1, 0.2, 0.3])
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'test.mp4')
expect(vi.mocked(fs.writeFileSync)).toHaveBeenCalled()
expect(vi.mocked(fs.readFileSync)).toHaveBeenCalled()
expect(vi.mocked(fs.unlinkSync)).toHaveBeenCalled()
expect(s3Service.getObject).toHaveBeenCalledWith(
'shumai',
'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
)
})
it('should generate text embedding successfully', async () => {
+9 -33
View File
@@ -310,7 +310,8 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
}
const parsedDuration = parseFloat(process.env.EMBEDDING_CHUNK_DURATION || '')
const chunkDuration = Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0
const chunkDuration =
Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0
return {
agent,
@@ -386,9 +387,7 @@ export async function generateImageEmbeddingActivity(
export interface GenerateVideoChunkEmbeddingParams {
teamId: string
assetKey: string
startTime: number
endTime: number
chunkKey: string
}
export async function generateVideoChunkEmbeddingActivity(
@@ -414,44 +413,21 @@ export async function generateVideoChunkEmbeddingActivity(
outputTokens: 0,
}
const { buffer: data } = await s3Service.getObject(
process.env.S3_BUCKET || 'shumai',
params.assetKey,
)
const tmpFile = path.join(os.tmpdir(), `video-chunk-source-${Date.now()}.mp4`)
fs.writeFileSync(tmpFile, data)
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`)
try {
await execFileAsync('ffmpeg', [
'-y',
'-i',
tmpFile,
'-ss',
params.startTime.toString(),
'-t',
(params.endTime - params.startTime).toString(),
'-c',
'copy',
chunkTmp,
])
const { buffer: data } = await s3Service.getObject(
process.env.S3_BUCKET || 'shumai',
params.chunkKey,
)
const chunkData = fs.readFileSync(chunkTmp)
fs.unlinkSync(chunkTmp)
const embVec = await generateMultimodalEmbedding(ai, chunkData, 'video/mp4')
const embVec = await generateMultimodalEmbedding(ai, data, 'video/mp4')
return {
embedding: embVec,
usage,
}
} catch (err) {
throw ApplicationFailure.create({
message: `Failed to generate video chunk embedding for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
message: `Failed to generate video chunk embedding for key ${params.chunkKey}: ${err instanceof Error ? err.message : String(err)}`,
nonRetryable: false,
})
} finally {
if (fs.existsSync(tmpFile)) fs.unlinkSync(tmpFile)
if (fs.existsSync(chunkTmp)) fs.unlinkSync(chunkTmp)
}
}
@@ -44,9 +44,25 @@ describe('Agent Embedding Workflow', () => {
getAgentWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getAgentWorkerQueueActivity',
}),
getTranscodeWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getTranscodeWorkerQueueActivity',
}),
downloadMediaToTmpActivity: Object.assign(vi.fn(), {
_activityName: 'downloadMediaToTmpActivity',
}),
cleanupTmpDirActivity: Object.assign(vi.fn(), {
_activityName: 'cleanupTmpDirActivity',
}),
transcodeVideoChunkActivity: Object.assign(vi.fn(), {
_activityName: 'transcodeVideoChunkActivity',
}),
deleteS3ObjectActivity: Object.assign(vi.fn(), {
_activityName: 'deleteS3ObjectActivity',
}),
}
mockActivities.getAgentWorkerQueueActivity.mockResolvedValue('agent_queue')
mockActivities.getTranscodeWorkerQueueActivity.mockResolvedValue('transcode_queue')
mockActivities.updateTaskStatusActivity.mockResolvedValue({})
mockActivities.createCommentActivity.mockResolvedValue({ id: 'comment-placeholder-id' })
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
@@ -66,6 +82,19 @@ describe('Agent Embedding Workflow', () => {
embedding: [0.3, 0.4],
usage: { inputTokens: 3, outputTokens: 3, model: 'gpt' },
})
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/test.mp4',
tmpDir: '/tmp/test-dir',
})
mockActivities.transcodeVideoChunkActivity.mockImplementation(
async (params: { assetId: string; startTime: number; endTime: number }) => {
return {
chunkKey: `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}.mp4`,
}
},
)
mockActivities.deleteS3ObjectActivity.mockResolvedValue(undefined)
mockActivities.cleanupTmpDirActivity.mockResolvedValue(undefined)
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities contains vi.fn mock functions which are cast to expected activity proxy types
vi.mocked(workflowUtils.getActivities).mockReturnValue(mockActivities as any)
@@ -156,8 +185,8 @@ describe('Agent Embedding Workflow', () => {
mediaType: 'image/png',
storageKey: { key: 'test.png' },
media: {
imageTranscodes: [{ key: 'test-transcoded.webp', isRaw: false, format: 'webp' }]
}
imageTranscodes: [{ key: 'test-transcoded.webp', isRaw: false, format: 'webp' }],
},
},
chunkDuration: 60.0,
})
@@ -210,27 +239,59 @@ describe('Agent Embedding Workflow', () => {
await agentEmbeddingMedia(task)
// Verify video chunk embedding generation (3 chunks: 0-60, 60-120, 120-150)
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenCalledTimes(3)
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(1, {
teamId: 't1',
// Verify video transcode queue discovery & download
expect(mockActivities.getTranscodeWorkerQueueActivity).toHaveBeenCalled()
expect(mockActivities.downloadMediaToTmpActivity).toHaveBeenCalledWith({
assetKey: 'test-transcoded.mp4',
})
// Verify video chunk slicing activity called 3 times on transcode queue
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(3)
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(1, {
assetId: 'a1',
filePath: '/tmp/test.mp4',
startTime: 0,
endTime: 60,
})
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, {
teamId: 't1',
assetKey: 'test-transcoded.mp4',
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
assetId: 'a1',
filePath: '/tmp/test.mp4',
startTime: 60,
endTime: 120,
})
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, {
teamId: 't1',
assetKey: 'test-transcoded.mp4',
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
assetId: 'a1',
filePath: '/tmp/test.mp4',
startTime: 120,
endTime: 150,
})
// Verify video chunk embedding generation (3 chunks) on agent queue
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenCalledTimes(3)
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(1, {
teamId: 't1',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
})
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, {
teamId: 't1',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-60-120.mp4',
})
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, {
teamId: 't1',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-120-150.mp4',
})
// Verify deletion of temporary chunks from S3
expect(mockActivities.deleteS3ObjectActivity).toHaveBeenCalledTimes(3)
expect(mockActivities.deleteS3ObjectActivity).toHaveBeenNthCalledWith(1, {
key: 'files/a1/tmp-embedding-chunks/chunk-0-60.mp4',
})
// Verify temp directory cleanup on transcode queue
expect(mockActivities.cleanupTmpDirActivity).toHaveBeenCalledWith({
tmpDir: '/tmp/test-dir',
})
// Verify usage update (3 chunks * 3 tokens = 9)
expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith({
taskId: task.id,
+68 -20
View File
@@ -1,6 +1,11 @@
import { ApplicationFailure } from '@temporalio/workflow'
import type { WorkflowTask } from '@shumai/db'
import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core'
import {
getActivities,
executeActivity,
TaskQueueAgent,
TaskQueueTranscode,
} from '@shumai/workflow-core'
export interface GeneratedEmbedding {
embedding: number[]
@@ -19,14 +24,25 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
createCommentActivity,
updateCommentActivity,
getAgentWorkerQueueActivity,
getTranscodeWorkerQueueActivity,
downloadMediaToTmpActivity,
cleanupTmpDirActivity,
transcodeVideoChunkActivity,
deleteS3ObjectActivity,
} = getActivities()
let placeholderCommentId: string | undefined
let agentWorkerQueue = ''
let transcodeWorkerQueue = ''
let tmpDir: string | undefined
try {
// 0. Discover queue
// 0. Discover queues
agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity)
transcodeWorkerQueue = await executeActivity(
TaskQueueTranscode,
getTranscodeWorkerQueueActivity,
)
// Update status to processing
await executeActivity(agentWorkerQueue, updateTaskStatusActivity, {
@@ -77,7 +93,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
if (media.videoTranscodes && media.videoTranscodes.length > 0) {
// Find first non-raw transcode or fallback to first transcode
const transcode =
media.videoTranscodes.find((t: PrismaJson.VideoTranscode) => !t.isRaw) || media.videoTranscodes[0]
media.videoTranscodes.find((t: PrismaJson.VideoTranscode) => !t.isRaw) ||
media.videoTranscodes[0]
if (transcode?.key) {
embeddingKey = transcode.key
}
@@ -86,7 +103,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
if (media.imageTranscodes && media.imageTranscodes.length > 0) {
// Find first non-raw transcode or fallback to first transcode
const transcode =
media.imageTranscodes.find((t: PrismaJson.ImageTranscode) => !t.isRaw) || media.imageTranscodes[0]
media.imageTranscodes.find((t: PrismaJson.ImageTranscode) => !t.isRaw) ||
media.imageTranscodes[0]
if (transcode?.key) {
embeddingKey = transcode.key
if (transcode.format) {
@@ -120,6 +138,13 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
usage.outputTokens += result.usage.outputTokens || 0
}
} else if (isVideo) {
// 1. Download full video to transcode worker temp space
const download = await executeActivity(transcodeWorkerQueue, downloadMediaToTmpActivity, {
assetKey: embeddingKey,
})
const { filePath } = download
tmpDir = download.tmpDir
// Get video duration
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const duration = (asset.media as any)?.duration || 0
@@ -129,26 +154,41 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
let end = start + limit
if (end > duration) end = duration
const chunkResult = await executeActivity(
agentWorkerQueue,
generateVideoChunkEmbeddingActivity,
{
teamId: task.teamId,
assetKey: embeddingKey,
startTime: start,
endTime: end,
},
)
results.push({
embedding: chunkResult.embedding,
// Step A: Slice chunk on transcode worker and upload to S3
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
assetId: asset.id,
filePath,
startTime: start,
endTime: end,
})
const { chunkKey } = chunkRes
if (chunkResult.usage) {
usage.inputTokens += chunkResult.usage.inputTokens || 0
usage.outputTokens += chunkResult.usage.outputTokens || 0
try {
// Step B: Generate embedding on agent worker
const chunkResult = await executeActivity(
agentWorkerQueue,
generateVideoChunkEmbeddingActivity,
{
teamId: task.teamId,
chunkKey,
},
)
results.push({
embedding: chunkResult.embedding,
startTime: start,
endTime: end,
})
if (chunkResult.usage) {
usage.inputTokens += chunkResult.usage.inputTokens || 0
usage.outputTokens += chunkResult.usage.outputTokens || 0
}
} finally {
// Step C: Delete temporary chunk from S3
await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, {
key: chunkKey,
})
}
}
}
@@ -206,6 +246,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
})
}
throw err
} finally {
if (tmpDir && transcodeWorkerQueue) {
try {
await executeActivity(transcodeWorkerQueue, cleanupTmpDirActivity, { tmpDir })
} catch (cleanupErr) {
console.error('Failed to cleanup transcode worker tmp dir:', cleanupErr)
}
}
}
}
@@ -4,6 +4,11 @@ import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { transcodeService } from '@shumai/core'
import * as child_process from 'child_process'
vi.mock('child_process', () => ({
execFile: vi.fn(),
}))
import {
getMediaInfoActivity,
overlayAnnotationsActivity,
@@ -13,6 +18,8 @@ import {
downloadMediaToTmpActivity,
transcodeImageActivity,
generateSpriteActivity,
transcodeVideoChunkActivity,
deleteS3ObjectActivity,
} from './transcode'
vi.mock('@shumai/core/src/s3/s3', () => ({
@@ -22,6 +29,7 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
headObject: vi.fn(),
presign: vi.fn(),
downloadToFile: vi.fn(),
deleteObject: vi.fn(),
},
}))
@@ -365,4 +373,56 @@ describe('Transcode Activities', () => {
).rejects.toThrowError(/Overlay annotations failed/)
})
})
describe('Video Chunk Activities', () => {
it('should transcode video chunk using ffmpeg and upload to S3', async () => {
// Mock child_process.execFile to simulate ffmpeg succeeding
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(file: string, args: string[], cb: any) => {
cb(null, { stdout: '', stderr: '' })
},
)
const res = await transcodeVideoChunkActivity({
assetId: 'asset-abc',
filePath: '/tmp/full-video.mp4',
startTime: 10,
endTime: 25,
})
// Verify ffmpeg command args
expect(child_process.execFile).toHaveBeenCalledWith(
'ffmpeg',
expect.arrayContaining([
'-y',
'-i',
'/tmp/full-video.mp4',
'-ss',
'10',
'-t',
'15',
'-c',
'copy',
]),
expect.any(Function),
)
// Verify S3 upload
expect(s3Service.putObject).toHaveBeenCalledWith(
'shumai',
expect.stringMatching(/^files\/asset-abc\/tmp-embedding-chunks\/chunk-10-25-.*\.mp4$/),
expect.any(Buffer),
expect.any(Number),
'video/mp4',
)
expect(res.chunkKey).toMatch(/^files\/asset-abc\/tmp-embedding-chunks\/chunk-10-25-.*\.mp4$/)
})
it('should delete S3 object successfully', async () => {
await deleteS3ObjectActivity({ key: 'files/test-key.mp4' })
expect(s3Service.deleteObject).toHaveBeenCalledWith('shumai', 'files/test-key.mp4')
})
})
})
@@ -5,6 +5,12 @@ import { metadataService } from '@shumai/core/src/metadata/metadata'
import { ApplicationFailure } from '@temporalio/activity'
import * as path from 'path'
import * as fs from 'fs'
import * as os from 'os'
import { execFile } from 'child_process'
import { promisify } from 'util'
import { ulid } from 'ulid'
const execFileAsync = promisify(execFile)
export interface GetMediaInfoActivityParams {
assetId: string
@@ -539,3 +545,59 @@ export async function createEmbeddingTaskIfEnabledActivity(
},
})
}
export interface TranscodeVideoChunkParams {
assetId: string
filePath: string
startTime: number
endTime: number
}
export async function transcodeVideoChunkActivity(
params: TranscodeVideoChunkParams,
): Promise<{ chunkKey: string }> {
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`)
try {
// Slice video segment using ffmpeg
await execFileAsync('ffmpeg', [
'-y',
'-i',
params.filePath,
'-ss',
params.startTime.toString(),
'-t',
(params.endTime - params.startTime).toString(),
'-c',
'copy',
chunkTmp,
])
const chunkData = fs.readFileSync(chunkTmp)
// Upload to S3
const bucket = process.env.S3_BUCKET || 'shumai'
const key = `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}-${ulid()}.mp4`
await s3Service.putObject(bucket, key, chunkData, chunkData.length, 'video/mp4')
return { chunkKey: key }
} catch (err) {
throw ApplicationFailure.create({
message: `Failed to transcode video chunk for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
nonRetryable: false,
})
} finally {
if (fs.existsSync(chunkTmp)) {
try {
fs.unlinkSync(chunkTmp)
} catch (e) {
console.error('Failed to cleanup local chunk file:', e)
}
}
}
}
export async function deleteS3ObjectActivity(params: { key: string }): Promise<void> {
const bucket = process.env.S3_BUCKET || 'shumai'
await s3Service.deleteObject(bucket, params.key)
}