mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-21 04:49:43 +00:00
Merge pull request #127 from shumaiOne/feat/semantic-search-chunk-overlap
feat(agent): implement overlapping video chunks for semantic search
This commit is contained in:
@@ -59,6 +59,7 @@ Configures whether background transcoding and AI agents are executed locally or
|
||||
| `WORKFLOW_EXECUTOR` | Engine choice. Use `local` for in-process database polling, or `temporal` for distributed workers. | `local` (default) or `temporal` |
|
||||
| `TEMPORAL_ADDRESS` | Host and port of the Temporal cluster gRPC service. | `localhost:7233` (default) |
|
||||
| `EMBEDDING_CHUNK_DURATION` | The duration in seconds for each video chunk when generating video embeddings. | `60.0` (default) |
|
||||
| `EMBEDDING_CHUNK_OVERLAP` | The duration in seconds to overlap video chunks when generating video embeddings. | `5.0` (default) |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -12,6 +12,11 @@ Traditional search relies on matching exact characters (e.g., searching "dog" wi
|
||||
Semantic search translates text queries and file contents into conceptual profiles. Similar ideas and visuals are mapped near each other.
|
||||
|
||||
1. **Index Generation**: When you upload an asset, Shumai's background agent scans the file's visual elements, text contents, or descriptions, translating them into a concept profile.
|
||||
- **Video Processing**: Shumai uses **gemini-embedding-2** for video indexing.
|
||||
- **Chunking**: For long videos, Shumai breaks them into chunks to ensure high-quality indexing. By default, chunks are **60 seconds** long with a **5-second overlap** to ensure no context is lost at the boundaries.
|
||||
- **Sampling**: The model processes a maximum of 32 frames per chunk: short chunks (≤32s) are sampled at 1 fps, while longer chunks are uniformly sampled to 32 frames.
|
||||
- **Optimization**: To optimize processing, Shumai automatically transcodes each chunk to **1 FPS** and **removes the audio track** before sending it to the embedding agent.
|
||||
- **Configuration**: Administrators can customize these settings using the `EMBEDDING_CHUNK_DURATION` and `EMBEDDING_CHUNK_OVERLAP` environment variables.
|
||||
2. **Concept Matching**: When you search with the "Semantic" toggle enabled, Shumai matches the conceptual meaning of your query against your assets.
|
||||
3. **Result**: Searching "sunset at the beach" will successfully find a video file of ocean waves at dusk, even if the filename is `IMG_4029.mp4`.
|
||||
|
||||
|
||||
@@ -302,41 +302,36 @@ describe('AI Database Activities Integration', () => {
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(60.0)
|
||||
expect(ctx.chunkOverlap).toBe(5.0)
|
||||
|
||||
// Test valid positive
|
||||
process.env.EMBEDDING_CHUNK_DURATION = '45.5'
|
||||
process.env.EMBEDDING_CHUNK_OVERLAP = '10.0'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(45.5)
|
||||
expect(ctx.chunkOverlap).toBe(10.0)
|
||||
|
||||
// Test negative
|
||||
process.env.EMBEDDING_CHUNK_DURATION = '-10.0'
|
||||
// Test negative/invalid for overlap
|
||||
process.env.EMBEDDING_CHUNK_OVERLAP = '-5.0'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(60.0)
|
||||
expect(ctx.chunkOverlap).toBe(5.0)
|
||||
|
||||
// Test zero
|
||||
process.env.EMBEDDING_CHUNK_DURATION = '0'
|
||||
process.env.EMBEDDING_CHUNK_OVERLAP = 'invalid'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(60.0)
|
||||
|
||||
// Test non-numeric
|
||||
process.env.EMBEDDING_CHUNK_DURATION = 'invalid'
|
||||
ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
expect(ctx.chunkDuration).toBe(60.0)
|
||||
expect(ctx.chunkOverlap).toBe(5.0)
|
||||
|
||||
// Cleanup env
|
||||
delete process.env.EMBEDDING_CHUNK_DURATION
|
||||
delete process.env.EMBEDDING_CHUNK_OVERLAP
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -315,10 +315,15 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
|
||||
const chunkDuration =
|
||||
Number.isFinite(parsedDuration) && parsedDuration > 0 ? parsedDuration : 60.0
|
||||
|
||||
const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '')
|
||||
const chunkOverlap =
|
||||
Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
|
||||
|
||||
return {
|
||||
agent,
|
||||
asset,
|
||||
chunkDuration,
|
||||
chunkOverlap,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
storageKey: { key: 'test.png' },
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
mockActivities.generateImageEmbeddingActivity.mockResolvedValue({
|
||||
embedding: [0.1, 0.2],
|
||||
@@ -189,6 +190,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
@@ -222,6 +224,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
@@ -256,13 +259,13 @@ describe('Agent Embedding Workflow', () => {
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
|
||||
assetId: 'a1',
|
||||
filePath: '/tmp/test.mp4',
|
||||
startTime: 60,
|
||||
endTime: 120,
|
||||
startTime: 55,
|
||||
endTime: 115,
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
|
||||
assetId: 'a1',
|
||||
filePath: '/tmp/test.mp4',
|
||||
startTime: 120,
|
||||
startTime: 110,
|
||||
endTime: 150,
|
||||
})
|
||||
|
||||
@@ -274,11 +277,11 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(2, {
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-60-120.mp4',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-55-115.mp4',
|
||||
})
|
||||
expect(mockActivities.generateVideoChunkEmbeddingActivity).toHaveBeenNthCalledWith(3, {
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-120-150.mp4',
|
||||
chunkKey: 'files/a1/tmp-embedding-chunks/chunk-110-150.mp4',
|
||||
})
|
||||
|
||||
// Verify deletion of temporary chunks from S3
|
||||
@@ -305,8 +308,8 @@ describe('Agent Embedding Workflow', () => {
|
||||
assetId: 'a1',
|
||||
embeddings: [
|
||||
{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 },
|
||||
{ embedding: [0.3, 0.4], startTime: 60, endTime: 120 },
|
||||
{ embedding: [0.3, 0.4], startTime: 120, endTime: 150 },
|
||||
{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 },
|
||||
{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 },
|
||||
],
|
||||
})
|
||||
})
|
||||
@@ -342,6 +345,7 @@ describe('Agent Embedding Workflow', () => {
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
@@ -391,4 +395,40 @@ describe('Agent Embedding Workflow', () => {
|
||||
output: { error: 'AI Service Unavailable' },
|
||||
})
|
||||
})
|
||||
|
||||
it('should not create redundant chunks when video exactly matches chunk duration', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 60.0,
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Verify only 1 chunk was created instead of 2 (0-60 and 55-60)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(1)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -71,7 +71,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const { asset, chunkDuration } = context as any
|
||||
const { asset, chunkDuration, chunkOverlap } = context as any
|
||||
|
||||
const isImage = asset.mediaType?.startsWith('image/')
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
@@ -149,10 +149,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const duration = (asset.media as any)?.duration || 0
|
||||
const limit = chunkDuration || 60.0
|
||||
const overlap = chunkOverlap ?? 5.0
|
||||
const step = Math.max(1, limit - overlap)
|
||||
|
||||
for (let start = 0.0; start < duration; start += limit) {
|
||||
for (let start = 0.0; start < duration; start += step) {
|
||||
let end = start + limit
|
||||
if (end > duration) end = duration
|
||||
if (end >= duration) {
|
||||
end = duration
|
||||
}
|
||||
|
||||
// Step A: Slice chunk on transcode worker and upload to S3
|
||||
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
|
||||
@@ -190,6 +194,11 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
key: chunkKey,
|
||||
})
|
||||
}
|
||||
|
||||
// If this chunk reached the end of the video, stop to avoid redundant overlapping chunks
|
||||
if (end >= duration) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -406,8 +406,15 @@ describe('Transcode Activities', () => {
|
||||
'10',
|
||||
'-t',
|
||||
'15',
|
||||
'-c',
|
||||
'copy',
|
||||
'-vf',
|
||||
'fps=1',
|
||||
'-c:v',
|
||||
'libx264',
|
||||
'-preset',
|
||||
'ultrafast',
|
||||
'-crf',
|
||||
'28',
|
||||
'-an',
|
||||
]),
|
||||
expect.any(Function),
|
||||
)
|
||||
|
||||
@@ -558,7 +558,7 @@ export async function transcodeVideoChunkActivity(
|
||||
): Promise<{ chunkKey: string }> {
|
||||
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}.mp4`)
|
||||
try {
|
||||
// Slice video segment using ffmpeg
|
||||
// Slice video segment using ffmpeg, force 1fps and remove audio for gemini-embedding-2 optimization
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
@@ -569,8 +569,15 @@ export async function transcodeVideoChunkActivity(
|
||||
params.startTime.toString(),
|
||||
'-t',
|
||||
(params.endTime - params.startTime).toString(),
|
||||
'-c',
|
||||
'copy',
|
||||
'-vf',
|
||||
'fps=1',
|
||||
'-c:v',
|
||||
'libx264',
|
||||
'-preset',
|
||||
'ultrafast',
|
||||
'-crf',
|
||||
'28',
|
||||
'-an',
|
||||
chunkTmp,
|
||||
])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user