From eaf2c867b3b05a07a44c29f12ace0e2bfb1c1ce3 Mon Sep 17 00:00:00 2001 From: yiling Date: Thu, 18 Jun 2026 19:45:58 +0800 Subject: [PATCH 1/4] feat(agent): implement overlapping video chunks for semantic search and update docs --- docs/configuration/env-variables.mdx | 1 + docs/using-shumai/semantic-search.mdx | 1 + packages/agent/src/activities/ai.test.ts | 23 ++++++++----------- packages/agent/src/activities/ai.ts | 5 ++++ .../src/workflows/agent-embedding.test.ts | 18 +++++++++------ .../agent/src/workflows/agent-embedding.ts | 6 +++-- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/docs/configuration/env-variables.mdx b/docs/configuration/env-variables.mdx index fbc9bd88..705e60b2 100644 --- a/docs/configuration/env-variables.mdx +++ b/docs/configuration/env-variables.mdx @@ -59,6 +59,7 @@ Configures whether background transcoding and AI agents are executed locally or | `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` | | `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) | | `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) | +| `EMBEDDING_CHUNK_OVERLAP` | The duration in seconds to overlap video chunks when generating video embeddings. | `5.0` (default) | --- diff --git a/docs/using-shumai/semantic-search.mdx b/docs/using-shumai/semantic-search.mdx index 8b308a51..589f582d 100644 --- a/docs/using-shumai/semantic-search.mdx +++ b/docs/using-shumai/semantic-search.mdx @@ -12,6 +12,7 @@ Traditional search relies on matching exact characters (e.g., searching "dog" wi Semantic search translates text queries and file contents into conceptual profiles. Similar ideas and visuals are mapped near each other. 1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile. + - **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing. This model processes a maximum of 32 frames per video: short videos (≤32s) are sampled at 1 fps, while longer videos are uniformly sampled to 32 frames. Note that audio tracks are not processed. 2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets. 3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`. diff --git a/packages/agent/src/activities/ai.test.ts b/packages/agent/src/activities/ai.test.ts index 09e2e385..66912080 100644 --- a/packages/agent/src/activities/ai.test.ts +++ b/packages/agent/src/activities/ai.test.ts @@ -302,41 +302,36 @@ describe('AI Database Activities Integration', () => { assetId: asset.id, }) expect(ctx.chunkDuration).toBe(60.0) + expect(ctx.chunkOverlap).toBe(5.0) // Test valid positive process.env.EMBEDDING_CHUNK_DURATION = '45.5' + process.env.EMBEDDING_CHUNK_OVERLAP = '10.0' ctx = await getEmbeddingContextActivity({ teamId: team.id, assetId: asset.id, }) expect(ctx.chunkDuration).toBe(45.5) + expect(ctx.chunkOverlap).toBe(10.0) - // Test negative - process.env.EMBEDDING_CHUNK_DURATION = '-10.0' + // Test negative/invalid for overlap + process.env.EMBEDDING_CHUNK_OVERLAP = '-5.0' ctx = await getEmbeddingContextActivity({ teamId: team.id, assetId: asset.id, }) - expect(ctx.chunkDuration).toBe(60.0) + expect(ctx.chunkOverlap).toBe(5.0) - // Test zero - process.env.EMBEDDING_CHUNK_DURATION = '0' + process.env.EMBEDDING_CHUNK_OVERLAP = 'invalid' ctx = await getEmbeddingContextActivity({ teamId: team.id, assetId: asset.id, }) - expect(ctx.chunkDuration).toBe(60.0) - - // Test non-numeric - process.env.EMBEDDING_CHUNK_DURATION = 'invalid' - ctx = await getEmbeddingContextActivity({ - teamId: team.id, - assetId: asset.id, - }) - expect(ctx.chunkDuration).toBe(60.0) + expect(ctx.chunkOverlap).toBe(5.0) // Cleanup env delete process.env.EMBEDDING_CHUNK_DURATION + delete process.env.EMBEDDING_CHUNK_OVERLAP }) }) diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index 85a270e1..df1b42c7 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -315,10 +315,15 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse const chunkDuration = Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0 + const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '') + const chunkOverlap = + Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0 + return { agent, asset, chunkDuration, + chunkOverlap, } } diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts index 43725acc..f628ff47 100644 --- a/packages/agent/src/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -73,6 +73,7 @@ describe('Agent Embedding Workflow', () => { storageKey: { key: 'test.png' }, }, chunkDuration: 60.0, + chunkOverlap: 5.0, }) mockActivities.generateImageEmbeddingActivity.mockResolvedValue({ embedding: [0.1, 0.2], @@ -189,6 +190,7 @@ describe('Agent Embedding Workflow', () => { }, }, chunkDuration: 60.0, + chunkOverlap: 5.0, }) const task = await prisma.workflowTask.create({ @@ -222,6 +224,7 @@ describe('Agent Embedding Workflow', () => { }, }, chunkDuration: 60.0, + chunkOverlap: 5.0, }) const task = await prisma.workflowTask.create({ @@ -256,13 +259,13 @@ describe('Agent Embedding Workflow', () => { expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, { assetId: 'a1', filePath: '/tmp/test.mp4', - startTime: 60, - endTime: 120, + startTime: 55, + endTime: 115, }) expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, { assetId: 'a1', filePath: '/tmp/test.mp4', - startTime: 120, + startTime: 110, endTime: 150, }) @@ -274,11 +277,11 @@ describe('Agent Embedding Workflow', () => { }) expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, { teamId: 't1', - chunkKey: 'files/a1/tmp-embedding-chunks/chunk-60-120.mp4', + chunkKey: 'files/a1/tmp-embedding-chunks/chunk-55-115.mp4', }) expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, { teamId: 't1', - chunkKey: 'files/a1/tmp-embedding-chunks/chunk-120-150.mp4', + chunkKey: 'files/a1/tmp-embedding-chunks/chunk-110-150.mp4', }) // Verify deletion of temporary chunks from S3 @@ -305,8 +308,8 @@ describe('Agent Embedding Workflow', () => { assetId: 'a1', embeddings: [ { embedding: [0.3, 0.4], startTime: 0, endTime: 60 }, - { embedding: [0.3, 0.4], startTime: 60, endTime: 120 }, - { embedding: [0.3, 0.4], startTime: 120, endTime: 150 }, + { embedding: [0.3, 0.4], startTime: 55, endTime: 115 }, + { embedding: [0.3, 0.4], startTime: 110, endTime: 150 }, ], }) }) @@ -342,6 +345,7 @@ describe('Agent Embedding Workflow', () => { }, }, chunkDuration: 60.0, + chunkOverlap: 5.0, }) const task = await prisma.workflowTask.create({ diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index 1c829455..17479843 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -71,7 +71,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const { asset, chunkDuration } = context as any + const { asset, chunkDuration, chunkOverlap } = context as any const isImage = asset.mediaType?.startsWith('image/') const isVideo = asset.mediaType?.startsWith('video/') @@ -149,8 +149,10 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // eslint-disable-next-line @typescript-eslint/no-explicit-any const duration = (asset.media as any)?.duration || 0 const limit = chunkDuration || 60.0 + const overlap = chunkOverlap ?? 5.0 + const step = Math.max(1, limit - overlap) - for (let start = 0.0; start < duration; start += limit) { + for (let start = 0.0; start < duration; start += step) { let end = start + limit if (end > duration) end = duration From 212ac3bc0e956a46a4806042035e12d421e4df90 Mon Sep 17 00:00:00 2001 From: yiling Date: Thu, 18 Jun 2026 19:46:50 +0800 Subject: [PATCH 2/4] docs(search): add detail about video chunking and configuration to semantic search docs --- docs/using-shumai/semantic-search.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/using-shumai/semantic-search.mdx b/docs/using-shumai/semantic-search.mdx index 589f582d..fe0c2cc7 100644 --- a/docs/using-shumai/semantic-search.mdx +++ b/docs/using-shumai/semantic-search.mdx @@ -12,7 +12,10 @@ Traditional search relies on matching exact characters (e.g., searching "dog" wi Semantic search translates text queries and file contents into conceptual profiles. Similar ideas and visuals are mapped near each other. 1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile. - - **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing. This model processes a maximum of 32 frames per video: short videos (≤32s) are sampled at 1 fps, while longer videos are uniformly sampled to 32 frames. Note that audio tracks are not processed. + - **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing. + - **Chunking**: For long videos, Shumai breaks them into chunks to ensure high-quality indexing. By default, chunks are **60 seconds** long with a **5-second overlap** to ensure no context is lost at the boundaries. + - **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames. Note that audio tracks are not processed. + - **Configuration**: Administrators can customize these settings using the `EMBEDDING_CHUNK_DURATION` and `EMBEDDING_CHUNK_OVERLAP` environment variables. 2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets. 3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`. From 718d9d5b512e8a98158729171a7489cda1166b4c Mon Sep 17 00:00:00 2001 From: yiling Date: Thu, 18 Jun 2026 19:53:19 +0800 Subject: [PATCH 3/4] feat(transcode): optimize video chunks for embedding by forcing 1 FPS and dropping audio --- docs/using-shumai/semantic-search.mdx | 3 ++- packages/transcode/src/activities/transcode.test.ts | 11 +++++++++-- packages/transcode/src/activities/transcode.ts | 13 ++++++++++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/using-shumai/semantic-search.mdx b/docs/using-shumai/semantic-search.mdx index fe0c2cc7..d4efa281 100644 --- a/docs/using-shumai/semantic-search.mdx +++ b/docs/using-shumai/semantic-search.mdx @@ -14,7 +14,8 @@ Semantic search translates text queries and file contents into conceptual profil 1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile. - **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing. - **Chunking**: For long videos, Shumai breaks them into chunks to ensure high-quality indexing. By default, chunks are **60 seconds** long with a **5-second overlap** to ensure no context is lost at the boundaries. - - **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames. Note that audio tracks are not processed. + - **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames. + - **Optimization**: To optimize processing, Shumai automatically transcodes each chunk to **1 FPS** and **removes the audio track** before sending it to the embedding agent. - **Configuration**: Administrators can customize these settings using the `EMBEDDING_CHUNK_DURATION` and `EMBEDDING_CHUNK_OVERLAP` environment variables. 2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets. 3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`. diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 8d9e9481..b4586ef6 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -406,8 +406,15 @@ describe('Transcode Activities', () => { '10', '-t', '15', - '-c', - 'copy', + '-vf', + 'fps=1', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-crf', + '28', + '-an', ]), expect.any(Function), ) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index fb4f22d2..2017a863 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -558,7 +558,7 @@ export async function transcodeVideoChunkActivity( ): Promise<{ chunkKey: string }> { const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`) try { - // Slice video segment using ffmpeg + // Slice video segment using ffmpeg, force 1fps and remove audio for gemini-embedding-2 optimization await execFileAsync('ffmpeg', [ '-y', '-loglevel', @@ -569,8 +569,15 @@ export async function transcodeVideoChunkActivity( params.startTime.toString(), '-t', (params.endTime - params.startTime).toString(), - '-c', - 'copy', + '-vf', + 'fps=1', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-crf', + '28', + '-an', chunkTmp, ]) From a197c83580710db7ca5c4d86570cecb04cc3e27d Mon Sep 17 00:00:00 2001 From: yiling Date: Thu, 18 Jun 2026 19:57:01 +0800 Subject: [PATCH 4/4] fix(agent): prevent redundant final video chunks in embedding workflow --- .../src/workflows/agent-embedding.test.ts | 36 +++++++++++++++++++ .../agent/src/workflows/agent-embedding.ts | 9 ++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts index f628ff47..c409159b 100644 --- a/packages/agent/src/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -395,4 +395,40 @@ describe('Agent Embedding Workflow', () => { output: { error: 'AI Service Unavailable' }, }) }) + + it('should not create redundant chunks when video exactly matches chunk duration', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'a1', + mediaType: 'video/mp4', + storageKey: { key: 'test.mp4' }, + media: { + duration: 60.0, + }, + }, + chunkDuration: 60.0, + chunkOverlap: 5.0, + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'a1', + teamId: 't1', + }, + }) + + await agentEmbeddingMedia(task) + + // Verify only 1 chunk was created instead of 2 (0-60 and 55-60) + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(1) + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith( + expect.objectContaining({ + startTime: 0, + endTime: 60, + }), + ) + }) }) diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index 17479843..07f77f50 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -154,7 +154,9 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { for (let start = 0.0; start < duration; start += step) { let end = start + limit - if (end > duration) end = duration + if (end >= duration) { + end = duration + } // Step A: Slice chunk on transcode worker and upload to S3 const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, { @@ -192,6 +194,11 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { key: chunkKey, }) } + + // If this chunk reached the end of the video, stop to avoid redundant overlapping chunks + if (end >= duration) { + break + } } }