From 1a46f631603e5484f2f237ab41cc920a27b5a3b5 Mon Sep 17 00:00:00 2001 From: Yiling-J Date: Fri, 16 Oct 2026 08:33:29 +0800 Subject: [PATCH] fix(agent): adaptively set video embedding frame count based on aspect ratio (#477) * fix(agent): adaptively set video embedding frame count based on aspect ratio Adjust sampled video frame counts between 15 and 14 based on the video's aspect ratio. Standard aspect ratios (1:1, 4:3, 3:4, 16:9, 9:16, 21:9) stay within the 2048 token signature bucket when using 15 frames, while non-standard or custom aspect ratios use 14 frames to guarantee that sequence length never exceeds 2048 tokens, preventing out-of-memory errors on the embedding server. * feat(agent): persist video chunk embeddings incrementally and resume on retry (#478) --- CHANGELOG.md | 1 + packages/agent/src/activities/ai.test.ts | 170 +++++++++++++++++- packages/agent/src/activities/ai.ts | 92 +++++++++- packages/agent/src/index.ts | 3 + .../src/workflows/agent-embedding.test.ts | 157 +++++++++++++++- .../agent/src/workflows/agent-embedding.ts | 111 +++++++++--- packages/core/src/search/text-embedding.ts | 2 +- 7 files changed, 496 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da7e3ab3..cd3743c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +* **agent**: Save video chunk embeddings incrementally to the database as each chunk finishes, preserving completed progress on failure and resuming from where it left off on retry * **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing ### Fixed diff --git a/packages/agent/src/activities/ai.test.ts b/packages/agent/src/activities/ai.test.ts index 6361bde3..ae32949c 100644 --- a/packages/agent/src/activities/ai.test.ts +++ b/packages/agent/src/activities/ai.test.ts @@ -4,8 +4,10 @@ import { extractAiMetadataActivity, getEmbeddingContextActivity, saveAssetEmbeddingsActivity, + deleteAssetEmbeddingsActivity, generateImageEmbeddingActivity, generateVideoChunkEmbeddingActivity, + getGemmaEmbeddingTargetFrameCount, } from './ai' import { s3Service } from '@shumai/core/src/s3/s3' import { transcodeService } from '@shumai/core' @@ -335,7 +337,84 @@ describe('AI Activities Unit Tests', () => { expect(mockFetch).toHaveBeenCalled() }) - it('should sample uniform 16 frames when chunk archive contains more than 16 frames', async () => { + describe('getGemmaEmbeddingTargetFrameCount', () => { + it('should return 15 for standard aspect ratios', () => { + // 16:9 + expect(getGemmaEmbeddingTargetFrameCount(1920, 1080)).toBe(15) + expect(getGemmaEmbeddingTargetFrameCount(1280, 720)).toBe(15) + expect(getGemmaEmbeddingTargetFrameCount(854, 480)).toBe(15) + // 9:16 + expect(getGemmaEmbeddingTargetFrameCount(1080, 1920)).toBe(15) + // 1:1 + expect(getGemmaEmbeddingTargetFrameCount(1080, 1080)).toBe(15) + expect(getGemmaEmbeddingTargetFrameCount(480, 480)).toBe(15) + // 4:3 + expect(getGemmaEmbeddingTargetFrameCount(1440, 1080)).toBe(15) + expect(getGemmaEmbeddingTargetFrameCount(640, 480)).toBe(15) + // 3:4 + expect(getGemmaEmbeddingTargetFrameCount(1080, 1440)).toBe(15) + // 21:9 + expect(getGemmaEmbeddingTargetFrameCount(2560, 1080)).toBe(15) + expect(getGemmaEmbeddingTargetFrameCount(3440, 1440)).toBe(15) + }) + + it('should return 14 for non-standard aspect ratios and missing/invalid dimensions', () => { + expect(getGemmaEmbeddingTargetFrameCount(1000, 600)).toBe(14) // 5:3 + expect(getGemmaEmbeddingTargetFrameCount(700, 500)).toBe(14) // 1.4:1 + expect(getGemmaEmbeddingTargetFrameCount(854, 672)).toBe(14) // ~1.27:1 + expect(getGemmaEmbeddingTargetFrameCount(undefined, undefined)).toBe(14) + expect(getGemmaEmbeddingTargetFrameCount(0, 0)).toBe(14) + expect(getGemmaEmbeddingTargetFrameCount(-100, 100)).toBe(14) + }) + }) + + it('should sample uniform 15 frames when chunk archive contains standard 16:9 frames', async () => { + process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai' + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + model: 'embeddinggemma-2', + data: [{ embedding: Array(768).fill(0.3) }], + // eslint-disable-next-line @typescript-eslint/naming-convention + usage: { prompt_tokens: 140 }, + }), + }) + vi.stubGlobal('fetch', mockFetch) + + const sampleJpg = await sharp({ + create: { + width: 160, + height: 90, + channels: 3, + background: { r: 0, g: 0, b: 0 }, + }, + }) + .jpeg() + .toBuffer() + + const frames: Record = {} + for (let i = 1; i <= 30; i++) { + frames[`frame_${String(i).padStart(4, '0')}.jpg`] = sampleJpg + } + const sampleArchive = new Bun.Archive(frames) + const archiveBytes = await sampleArchive.bytes() + + vi.mocked(s3Service.getObject).mockResolvedValue({ + buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength), + contentType: 'application/x-tar', + } as unknown as { buffer: Buffer; contentType: string }) + + const res = await generateVideoChunkEmbeddingActivity({ + teamId: 't1', + chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar', + }) + + expect(res.embedding).toHaveLength(768) + const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body) + expect(requestBody.input).toHaveLength(15) + }) + + it('should fallback to 14 frames when chunk archive contains non-standard or unparseable frames', async () => { process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai' const mockFetch = vi.fn().mockResolvedValue({ ok: true, @@ -367,7 +446,7 @@ describe('AI Activities Unit Tests', () => { expect(res.embedding).toHaveLength(768) const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body) - expect(requestBody.input).toHaveLength(16) + expect(requestBody.input).toHaveLength(14) }) it('should throw non-retryable ApplicationFailure when chunk archive contains no valid frames', async () => { @@ -642,6 +721,45 @@ describe('AI Database Activities Integration', () => { delete process.env.EMBEDDING_CHUNK_DURATION delete process.env.EMBEDDING_CHUNK_OVERLAP }) + + it('should return existing embeddings for the asset', async () => { + // Ensure agent exists + await prisma.agent.upsert({ + where: { id: user.id }, + update: {}, + create: { + id: user.id, + teamId: team.id, + type: 'embedding', + enabled: true, + config: { provider: 'openai', model: 'gpt-4' }, + }, + }) + + await saveAssetEmbeddingsActivity({ + assetId: asset.id, + embeddings: [ + { embedding: Array(1536).fill(0.01), startTime: 0.0, endTime: 60.0 }, + { embedding: Array(768).fill(0.02), startTime: 55.0, endTime: 115.0 }, + ], + }) + + const ctx = await getEmbeddingContextActivity({ + teamId: team.id, + assetId: asset.id, + }) + + expect(ctx.existingEmbeddings).toHaveLength(2) + const chunk1 = ctx.existingEmbeddings.find((e) => e.startTime === 0) + expect(chunk1).toBeDefined() + expect(chunk1?.has1536).toBe(true) + expect(chunk1?.has768).toBe(false) + + const chunk2 = ctx.existingEmbeddings.find((e) => e.startTime === 55) + expect(chunk2).toBeDefined() + expect(chunk2?.has1536).toBe(false) + expect(chunk2?.has768).toBe(true) + }) }) describe('saveAssetEmbeddingsActivity', () => { @@ -709,4 +827,52 @@ describe('AI Database Activities Integration', () => { expect(dbEmbeddings[1].has1536).toBe(false) }) }) + + describe('deleteAssetEmbeddingsActivity', () => { + it('should delete specific embeddings by id', async () => { + await saveAssetEmbeddingsActivity({ + assetId: asset.id, + embeddings: [ + { embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 }, + { embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 }, + ], + }) + + const before = await prisma.assetEmbedding.findMany({ + where: { assetId: asset.id }, + }) + expect(before).toHaveLength(2) + + const targetId = before[0].id + await deleteAssetEmbeddingsActivity({ + assetId: asset.id, + ids: [targetId], + }) + + const after = await prisma.assetEmbedding.findMany({ + where: { assetId: asset.id }, + }) + expect(after).toHaveLength(1) + expect(after[0].id).not.toBe(targetId) + }) + + it('should delete all embeddings for an asset when ids is not provided', async () => { + await saveAssetEmbeddingsActivity({ + assetId: asset.id, + embeddings: [ + { embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 }, + { embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 }, + ], + }) + + await deleteAssetEmbeddingsActivity({ + assetId: asset.id, + }) + + const after = await prisma.assetEmbedding.findMany({ + where: { assetId: asset.id }, + }) + expect(after).toHaveLength(0) + }) + }) }) diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index 5dc68e02..c0412599 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -250,12 +250,49 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0 const embeddingProvider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2' + const existingEmbeddings = await prisma.$queryRaw` + SELECT + id, + start_time as "startTime", + end_time as "endTime", + (embedding IS NOT NULL) as "has1536", + (embedding_768 IS NOT NULL) as "has768" + FROM asset_embeddings + WHERE asset_id = ${params.assetId} + ` + return { agent, asset, chunkDuration, chunkOverlap, embeddingProvider, + existingEmbeddings, + } +} + +export interface ExistingAssetEmbeddingInfo { + id: string + startTime: number | null + endTime: number | null + has1536: boolean + has768: boolean +} + +export async function deleteAssetEmbeddingsActivity(params: { assetId: string; ids?: string[] }) { + if (params.ids && params.ids.length > 0) { + await prisma.assetEmbedding.deleteMany({ + where: { + assetId: params.assetId, + id: { in: params.ids }, + }, + }) + } else { + await prisma.assetEmbedding.deleteMany({ + where: { + assetId: params.assetId, + }, + }) } } @@ -360,6 +397,38 @@ export async function generateImageEmbeddingActivity( } } +/** + * Standard aspect ratios for video embedding: + * 1:1, 4:3, 3:4, 16:9, 9:16, 21:9 (including ultrawide 64:27), and 9:21. + * + * For these standard ratios, 15 frames guarantees that total input tokens stay strictly + * within the 2048 token signature bucket (<= 1982 tokens). For non-standard or custom + * aspect ratios (e.g. 5:3, 1.4:1), 14 frames is used to mathematically guarantee + * that sequence length never exceeds 2048 tokens (<= 1990 tokens), preventing OOM errors. + */ +export function getGemmaEmbeddingTargetFrameCount(width?: number, height?: number): number { + if (!width || !height || width <= 0 || height <= 0) { + return 14 + } + + const ratio = width / height + const standardRatios = [ + 1 / 1, // 1:1 + 4 / 3, // 4:3 + 3 / 4, // 3:4 + 16 / 9, // 16:9 + 9 / 16, // 9:16 + 21 / 9, // 21:9 + 64 / 27, // 21:9 ultrawide (2560x1080, 3440x1440) + 9 / 21, // 9:21 + 27 / 64, // 9:21 ultrawide portrait + ] + + // Allow up to 3% relative tolerance for minor encoder rounding (e.g. 854x480 vs 16:9) + const isStandard = standardRatios.some((std) => Math.abs(ratio - std) / std <= 0.03) + return isStandard ? 15 : 14 +} + export interface GenerateVideoChunkEmbeddingParams { teamId: string chunkKey: string @@ -402,10 +471,27 @@ export async function generateVideoChunkEmbeddingActivity( }) } + let targetFrameCount = 14 + try { + const firstFile = filesMap.get(frameFiles[0]) + if (firstFile) { + const firstFileBytes = await firstFile.bytes() + if (firstFileBytes.byteLength > 0) { + const meta = await sharp(Buffer.from(firstFileBytes)).metadata() + targetFrameCount = getGemmaEmbeddingTargetFrameCount(meta.width, meta.height) + } + } + } catch { + targetFrameCount = 14 + } + let selectedFrames = frameFiles - if (selectedFrames.length > 16) { - const step = (selectedFrames.length - 1) / 15 - selectedFrames = Array.from({ length: 16 }, (_, i) => selectedFrames[Math.round(i * step)]) + if (selectedFrames.length > targetFrameCount) { + const step = (selectedFrames.length - 1) / (targetFrameCount - 1) + selectedFrames = Array.from( + { length: targetFrameCount }, + (_, i) => selectedFrames[Math.round(i * step)], + ) } const input: OpenAiEmbeddingItem[] = await Promise.all( diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 1a947981..2270a1af 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -626,9 +626,12 @@ export { generateImageEmbeddingActivity, generateTextEmbeddingActivity, generateVideoChunkEmbeddingActivity, + saveAssetEmbeddingsActivity, + deleteAssetEmbeddingsActivity, type ExtractAiMetadataParams, type GeneratedEmbedding, type GenerateTextEmbeddingParams, + type ExistingAssetEmbeddingInfo, } from './activities/ai' export * from './database-session-storage' export * from './context/serialize-context' diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts index ecba82f2..eca87f3a 100644 --- a/packages/agent/src/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -38,6 +38,9 @@ describe('Agent Embedding Workflow', () => { saveAssetEmbeddingsActivity: Object.assign(vi.fn(), { _activityName: 'saveAssetEmbeddingsActivity', }), + deleteAssetEmbeddingsActivity: Object.assign(vi.fn(), { + _activityName: 'deleteAssetEmbeddingsActivity', + }), updateTaskUsageActivity: Object.assign(vi.fn(), { _activityName: 'updateTaskUsageActivity' }), createCommentActivity: Object.assign(vi.fn(), { _activityName: 'createCommentActivity' }), updateCommentActivity: Object.assign(vi.fn(), { _activityName: 'updateCommentActivity' }), @@ -360,14 +363,19 @@ describe('Agent Embedding Workflow', () => { model: 'gemini-embedding-2', }) - // Verify embeddings saved - expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({ + // Verify embeddings saved per chunk immediately + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(3) + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(1, { assetId: 'a1', - embeddings: [ - { embedding: [0.3, 0.4], startTime: 0, endTime: 60 }, - { embedding: [0.3, 0.4], startTime: 55, endTime: 115 }, - { embedding: [0.3, 0.4], startTime: 110, endTime: 150 }, - ], + embeddings: [{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 }], + }) + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(2, { + assetId: 'a1', + embeddings: [{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 }], + }) + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(3, { + assetId: 'a1', + embeddings: [{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 }], }) }) @@ -453,6 +461,49 @@ describe('Agent Embedding Workflow', () => { }) }) + it('should save completed chunks to DB immediately before subsequent chunk failure', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'a1', + mediaType: 'video/mp4', + storageKey: { key: 'test.mp4' }, + media: { + duration: 150.0, + videoTranscodes: [{ key: 'test-transcoded.mp4' }], + }, + }, + chunkDuration: 60.0, + chunkOverlap: 5.0, + }) + + // Chunk 1 succeeds, Chunk 2 fails + mockActivities.generateVideoChunkEmbeddingActivity + .mockResolvedValueOnce({ + embedding: [0.1, 0.2], + usage: { inputTokens: 3, outputTokens: 3, model: 'gemini-embedding-2' }, + }) + .mockRejectedValueOnce(new Error('Rate limit exceeded at chunk 2')) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'a1', + teamId: 't1', + }, + }) + + await expect(agentEmbeddingMedia(task)).rejects.toThrow('Rate limit exceeded at chunk 2') + + // Chunk 1 MUST have been saved to the database before chunk 2 failed + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(1) + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({ + assetId: 'a1', + embeddings: [{ embedding: [0.1, 0.2], startTime: 0, endTime: 60 }], + }) + }) + it('should not create redundant chunks when video exactly matches chunk duration', async () => { mockActivities.getEmbeddingContextActivity.mockResolvedValue({ agent: { id: 'b1' }, @@ -529,4 +580,96 @@ describe('Agent Embedding Workflow', () => { }), ) }) + + it('should skip already completed chunks when resuming a video embedding task', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'a1', + mediaType: 'video/mp4', + storageKey: { key: 'test.mp4' }, + media: { + duration: 150.0, + videoTranscodes: [{ key: 'test-transcoded.mp4' }], + }, + }, + chunkDuration: 60.0, + chunkOverlap: 5.0, + existingEmbeddings: [ + { + id: 'emb-1', + startTime: 0, + endTime: 60, + has1536: true, + has768: false, + }, + ], + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'a1', + teamId: 't1', + }, + }) + + await agentEmbeddingMedia(task) + + // Chunk 1 (0-60) should be skipped; only chunk 2 (55-115) and chunk 3 (110-150) should be transcoded and embedded + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(2) + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ startTime: 55, endTime: 115 }), + ) + expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ startTime: 110, endTime: 150 }), + ) + + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(2) + }) + + it('should purge incompatible embeddings before processing', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'a1', + mediaType: 'image/png', + storageKey: { key: 'test.png' }, + }, + embeddingProvider: 'gemma-embedding-2-openai', + existingEmbeddings: [ + { + id: 'old-gemini-emb', + startTime: null, + endTime: null, + has1536: true, + has768: false, + }, + ], + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'a1', + teamId: 't1', + }, + }) + + await agentEmbeddingMedia(task) + + // Should purge incompatible old-gemini-emb + expect(mockActivities.deleteAssetEmbeddingsActivity).toHaveBeenCalledWith({ + assetId: 'a1', + ids: ['old-gemini-emb'], + }) + + // Should generate and save new embedding + expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalled() + expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalled() + }) }) diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index 376f8485..1bb51d0a 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -13,6 +13,14 @@ export interface GeneratedEmbedding { endTime?: number } +export interface ExistingAssetEmbeddingInfo { + id: string + startTime: number | null + endTime: number | null + has1536: boolean + has768: boolean +} + export async function agentEmbeddingMedia(task: WorkflowTask): Promise { const { updateTaskStatusActivity, @@ -20,6 +28,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { generateImageEmbeddingActivity, generateVideoChunkEmbeddingActivity, saveAssetEmbeddingsActivity, + deleteAssetEmbeddingsActivity, updateTaskUsageActivity, createCommentActivity, updateCommentActivity, @@ -62,15 +71,41 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } // 1. Fetch Agent Context - const context = await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, { + const context = (await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, { teamId: task.teamId, assetId: task.assetId, - }) + })) as unknown as { + asset: { + id: string + mediaType: string | null + storageKey?: { key: string } | null + media?: unknown + } + chunkDuration: number + chunkOverlap: number + embeddingProvider: string + existingEmbeddings?: ExistingAssetEmbeddingInfo[] + } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const { asset, chunkDuration, chunkOverlap, embeddingProvider } = context as any + const { asset, chunkDuration, chunkOverlap, embeddingProvider, existingEmbeddings } = context const isGemmaProvider = embeddingProvider === 'gemma-embedding-2-openai' + const existing = existingEmbeddings || [] + + // Purge incompatible embeddings (e.g. 1536d when Gemma is used, or 768d when Gemini is used) + const incompatibleIds = existing + .filter((e) => (isGemmaProvider ? e.has1536 || !e.has768 : e.has768 || !e.has1536)) + .map((e) => e.id) + + if (incompatibleIds.length > 0) { + await executeActivity(agentWorkerQueue, deleteAssetEmbeddingsActivity, { + assetId: task.assetId, + ids: incompatibleIds, + }) + } + + const validExistingChunks = existing.filter((e) => !incompatibleIds.includes(e.id)) + const mediaInfo = asset.media as PrismaJson.MediaInfo | null const is3dWithViews = mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0 @@ -114,7 +149,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true }) } - const results: GeneratedEmbedding[] = [] const usage = { model: isGemmaProvider ? 'gemma-embedding-2-openai' : 'gemini-embedding-2', inputTokens: 0, @@ -122,15 +156,23 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } if (isImage) { - const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, { - teamId: task.teamId, - assetKey: embeddingKey, - mediaType: embeddingMediaType, - }) - results.push({ embedding: result.embedding }) - if (result.usage) { - usage.inputTokens += result.usage.inputTokens || 0 - usage.outputTokens += result.usage.outputTokens || 0 + const alreadyEmbedded = validExistingChunks.some( + (e) => e.startTime === null && e.endTime === null, + ) + if (!alreadyEmbedded) { + const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, { + teamId: task.teamId, + assetKey: embeddingKey, + mediaType: embeddingMediaType, + }) + await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, { + assetId: task.assetId, + embeddings: [{ embedding: result.embedding }], + }) + if (result.usage) { + usage.inputTokens += result.usage.inputTokens || 0 + usage.outputTokens += result.usage.outputTokens || 0 + } } } else if (isVideo) { // Get video duration @@ -146,6 +188,23 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { end = duration } + // Check if this chunk is already completed + const chunkExists = validExistingChunks.some( + (c) => + c.startTime !== null && + c.endTime !== null && + Math.abs(c.startTime - start) < 0.1 && + Math.abs(c.endTime - end) < 0.1, + ) + + if (chunkExists) { + // If this chunk reached the end of the video, stop + if (end >= duration) { + break + } + continue + } + // Step A: Slice chunk on transcode worker and upload to S3 const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, { assetId: asset.id, @@ -167,10 +226,16 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { }, ) - results.push({ - embedding: chunkResult.embedding, - startTime: start, - endTime: end, + // Step C: Save chunk embedding to database immediately + await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, { + assetId: task.assetId, + embeddings: [ + { + embedding: chunkResult.embedding, + startTime: start, + endTime: end, + }, + ], }) if (chunkResult.usage) { @@ -178,7 +243,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { usage.outputTokens += chunkResult.usage.outputTokens || 0 } } finally { - // Step C: Delete temporary chunk from S3 + // Step D: Delete temporary chunk from S3 await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, { key: chunkKey, }) @@ -191,14 +256,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } } - // Save computed embeddings - if (results.length > 0) { - await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, { - assetId: task.assetId, - embeddings: results, - }) - } - // Update Usage await executeActivity(agentWorkerQueue, updateTaskUsageActivity, { taskId: task.id, diff --git a/packages/core/src/search/text-embedding.ts b/packages/core/src/search/text-embedding.ts index 99f4fc01..271f9029 100644 --- a/packages/core/src/search/text-embedding.ts +++ b/packages/core/src/search/text-embedding.ts @@ -63,7 +63,7 @@ export class TextEmbeddingService { } } - const signal = options?.signal ?? AbortSignal.timeout(5000) + const signal = options?.signal ?? AbortSignal.timeout(15000) let result: TextEmbeddingResult if (provider === 'gemma-embedding-2-openai') {