Merge pull request #127 from shumaiOne/feat/semantic-search-chunk-overlap

feat(agent): implement overlapping video chunks for semantic search
This commit is contained in:
Yiling-J
2026-06-18 20:02:26 +08:00
committed by GitHub
8 changed files with 98 additions and 29 deletions
+1
View File
@@ -59,6 +59,7 @@ Configures whether background transcoding and AI agents are executed locally or
| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` |
| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) |
| `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) |
| `EMBEDDING_CHUNK_OVERLAP` | The duration in seconds to overlap video chunks when generating video embeddings. | `5.0` (default) |
---
+5
View File
@@ -12,6 +12,11 @@ Traditional search relies on matching exact characters (e.g., searching "dog" wi
Semantic search translates text queries and file contents into conceptual profiles. Similar ideas and visuals are mapped near each other.
1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile.
- **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing.
- **Chunking**: For long videos, Shumai breaks them into chunks to ensure high-quality indexing. By default, chunks are **60 seconds** long with a **5-second overlap** to ensure no context is lost at the boundaries.
- **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames.
- **Optimization**: To optimize processing, Shumai automatically transcodes each chunk to **1 FPS** and **removes the audio track** before sending it to the embedding agent.
- **Configuration**: Administrators can customize these settings using the `EMBEDDING_CHUNK_DURATION` and `EMBEDDING_CHUNK_OVERLAP` environment variables.
2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets.
3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`.
+9 -14
View File
@@ -302,41 +302,36 @@ describe('AI Database Activities Integration', () => {
assetId: asset.id,
})
expect(ctx.chunkDuration).toBe(60.0)
expect(ctx.chunkOverlap).toBe(5.0)
// Test valid positive
process.env.EMBEDDING_CHUNK_DURATION = '45.5'
process.env.EMBEDDING_CHUNK_OVERLAP = '10.0'
ctx = await getEmbeddingContextActivity({
teamId: team.id,
assetId: asset.id,
})
expect(ctx.chunkDuration).toBe(45.5)
expect(ctx.chunkOverlap).toBe(10.0)
// Test negative
process.env.EMBEDDING_CHUNK_DURATION = '-10.0'
// Test negative/invalid for overlap
process.env.EMBEDDING_CHUNK_OVERLAP = '-5.0'
ctx = await getEmbeddingContextActivity({
teamId: team.id,
assetId: asset.id,
})
expect(ctx.chunkDuration).toBe(60.0)
expect(ctx.chunkOverlap).toBe(5.0)
// Test zero
process.env.EMBEDDING_CHUNK_DURATION = '0'
process.env.EMBEDDING_CHUNK_OVERLAP = 'invalid'
ctx = await getEmbeddingContextActivity({
teamId: team.id,
assetId: asset.id,
})
expect(ctx.chunkDuration).toBe(60.0)
// Test non-numeric
process.env.EMBEDDING_CHUNK_DURATION = 'invalid'
ctx = await getEmbeddingContextActivity({
teamId: team.id,
assetId: asset.id,
})
expect(ctx.chunkDuration).toBe(60.0)
expect(ctx.chunkOverlap).toBe(5.0)
// Cleanup env
delete process.env.EMBEDDING_CHUNK_DURATION
delete process.env.EMBEDDING_CHUNK_OVERLAP
})
})
+5
View File
@@ -315,10 +315,15 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
const chunkDuration =
Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0
const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '')
const chunkOverlap =
Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
return {
agent,
asset,
chunkDuration,
chunkOverlap,
}
}
@@ -73,6 +73,7 @@ describe('Agent Embedding Workflow', () => {
storageKey: { key: 'test.png' },
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
mockActivities.generateImageEmbeddingActivity.mockResolvedValue({
embedding: [0.1, 0.2],
@@ -189,6 +190,7 @@ describe('Agent Embedding Workflow', () => {
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
const task = await prisma.workflowTask.create({
@@ -222,6 +224,7 @@ describe('Agent Embedding Workflow', () => {
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
const task = await prisma.workflowTask.create({
@@ -256,13 +259,13 @@ describe('Agent Embedding Workflow', () => {
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
assetId: 'a1',
filePath: '/tmp/test.mp4',
startTime: 60,
endTime: 120,
startTime: 55,
endTime: 115,
})
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
assetId: 'a1',
filePath: '/tmp/test.mp4',
startTime: 120,
startTime: 110,
endTime: 150,
})
@@ -274,11 +277,11 @@ describe('Agent Embedding Workflow', () => {
})
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, {
teamId: 't1',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-60-120.mp4',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-55-115.mp4',
})
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, {
teamId: 't1',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-120-150.mp4',
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-110-150.mp4',
})
// Verify deletion of temporary chunks from S3
@@ -305,8 +308,8 @@ describe('Agent Embedding Workflow', () => {
assetId: 'a1',
embeddings: [
{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 },
{ embedding: [0.3, 0.4], startTime: 60, endTime: 120 },
{ embedding: [0.3, 0.4], startTime: 120, endTime: 150 },
{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 },
{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 },
],
})
})
@@ -342,6 +345,7 @@ describe('Agent Embedding Workflow', () => {
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
const task = await prisma.workflowTask.create({
@@ -391,4 +395,40 @@ describe('Agent Embedding Workflow', () => {
output: { error: 'AI Service Unavailable' },
})
})
it('should not create redundant chunks when video exactly matches chunk duration', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'a1',
mediaType: 'video/mp4',
storageKey: { key: 'test.mp4' },
media: {
duration: 60.0,
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'a1',
teamId: 't1',
},
})
await agentEmbeddingMedia(task)
// Verify only 1 chunk was created instead of 2 (0-60 and 55-60)
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(1)
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith(
expect.objectContaining({
startTime: 0,
endTime: 60,
}),
)
})
})
@@ -71,7 +71,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
})
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const { asset, chunkDuration } = context as any
const { asset, chunkDuration, chunkOverlap } = context as any
const isImage = asset.mediaType?.startsWith('image/')
const isVideo = asset.mediaType?.startsWith('video/')
@@ -149,10 +149,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const duration = (asset.media as any)?.duration || 0
const limit = chunkDuration || 60.0
const overlap = chunkOverlap ?? 5.0
const step = Math.max(1, limit - overlap)
for (let start = 0.0; start < duration; start += limit) {
for (let start = 0.0; start < duration; start += step) {
let end = start + limit
if (end > duration) end = duration
if (end >= duration) {
end = duration
}
// Step A: Slice chunk on transcode worker and upload to S3
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
@@ -190,6 +194,11 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
key: chunkKey,
})
}
// If this chunk reached the end of the video, stop to avoid redundant overlapping chunks
if (end >= duration) {
break
}
}
}
@@ -406,8 +406,15 @@ describe('Transcode Activities', () => {
'10',
'-t',
'15',
'-c',
'copy',
'-vf',
'fps=1',
'-c:v',
'libx264',
'-preset',
'ultrafast',
'-crf',
'28',
'-an',
]),
expect.any(Function),
)
+10 -3
View File
@@ -558,7 +558,7 @@ export async function transcodeVideoChunkActivity(
): Promise<{ chunkKey: string }> {
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`)
try {
// Slice video segment using ffmpeg
// Slice video segment using ffmpeg, force 1fps and remove audio for gemini-embedding-2 optimization
await execFileAsync('ffmpeg', [
'-y',
'-loglevel',
@@ -569,8 +569,15 @@ export async function transcodeVideoChunkActivity(
params.startTime.toString(),
'-t',
(params.endTime - params.startTime).toString(),
'-c',
'copy',
'-vf',
'fps=1',
'-c:v',
'libx264',
'-preset',
'ultrafast',
'-crf',
'28',
'-an',
chunkTmp,
])