fix(agent): adaptively set video embedding frame count based on aspect ratio (#477)

* fix(agent): adaptively set video embedding frame count based on aspect ratio

Adjust sampled video frame counts between 15 and 14 based on the video's aspect ratio. Standard aspect ratios (1:1, 4:3, 3:4, 16:9, 9:16, 21:9) stay within the 2048 token signature bucket when using 15 frames, while non-standard or custom aspect ratios use 14 frames to guarantee that sequence length never exceeds 2048 tokens, preventing out-of-memory errors on the embedding server.

* feat(agent): persist video chunk embeddings incrementally and resume on retry (#478)
This commit is contained in:
Yiling-J
2026-10-16 08:36:51 +08:00
committed by yiling
parent 6cc90bf973
commit 1a46f63160
7 changed files with 496 additions and 40 deletions
+1
View File
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Added
* **agent**: Save video chunk embeddings incrementally to the database as each chunk finishes, preserving completed progress on failure and resuming from where it left off on retry
* **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing
### Fixed
+168 -2
View File
@@ -4,8 +4,10 @@ import {
extractAiMetadataActivity,
getEmbeddingContextActivity,
saveAssetEmbeddingsActivity,
deleteAssetEmbeddingsActivity,
generateImageEmbeddingActivity,
generateVideoChunkEmbeddingActivity,
getGemmaEmbeddingTargetFrameCount,
} from './ai'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
@@ -335,7 +337,84 @@ describe('AI Activities Unit Tests', () => {
expect(mockFetch).toHaveBeenCalled()
})
it('should sample uniform 16 frames when chunk archive contains more than 16 frames', async () => {
describe('getGemmaEmbeddingTargetFrameCount', () => {
it('should return 15 for standard aspect ratios', () => {
// 16:9
expect(getGemmaEmbeddingTargetFrameCount(1920, 1080)).toBe(15)
expect(getGemmaEmbeddingTargetFrameCount(1280, 720)).toBe(15)
expect(getGemmaEmbeddingTargetFrameCount(854, 480)).toBe(15)
// 9:16
expect(getGemmaEmbeddingTargetFrameCount(1080, 1920)).toBe(15)
// 1:1
expect(getGemmaEmbeddingTargetFrameCount(1080, 1080)).toBe(15)
expect(getGemmaEmbeddingTargetFrameCount(480, 480)).toBe(15)
// 4:3
expect(getGemmaEmbeddingTargetFrameCount(1440, 1080)).toBe(15)
expect(getGemmaEmbeddingTargetFrameCount(640, 480)).toBe(15)
// 3:4
expect(getGemmaEmbeddingTargetFrameCount(1080, 1440)).toBe(15)
// 21:9
expect(getGemmaEmbeddingTargetFrameCount(2560, 1080)).toBe(15)
expect(getGemmaEmbeddingTargetFrameCount(3440, 1440)).toBe(15)
})
it('should return 14 for non-standard aspect ratios and missing/invalid dimensions', () => {
expect(getGemmaEmbeddingTargetFrameCount(1000, 600)).toBe(14) // 5:3
expect(getGemmaEmbeddingTargetFrameCount(700, 500)).toBe(14) // 1.4:1
expect(getGemmaEmbeddingTargetFrameCount(854, 672)).toBe(14) // ~1.27:1
expect(getGemmaEmbeddingTargetFrameCount(undefined, undefined)).toBe(14)
expect(getGemmaEmbeddingTargetFrameCount(0, 0)).toBe(14)
expect(getGemmaEmbeddingTargetFrameCount(-100, 100)).toBe(14)
})
})
it('should sample uniform 15 frames when chunk archive contains standard 16:9 frames', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.3) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 140 },
}),
})
vi.stubGlobal('fetch', mockFetch)
const sampleJpg = await sharp({
create: {
width: 160,
height: 90,
channels: 3,
background: { r: 0, g: 0, b: 0 },
},
})
.jpeg()
.toBuffer()
const frames: Record<string, Buffer> = {}
for (let i = 1; i <= 30; i++) {
frames[`frame_${String(i).padStart(4, '0')}.jpg`] = sampleJpg
}
const sampleArchive = new Bun.Archive(frames)
const archiveBytes = await sampleArchive.bytes()
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
contentType: 'application/x-tar',
} as unknown as { buffer: Buffer; contentType: string })
const res = await generateVideoChunkEmbeddingActivity({
teamId: 't1',
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
})
expect(res.embedding).toHaveLength(768)
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
expect(requestBody.input).toHaveLength(15)
})
it('should fallback to 14 frames when chunk archive contains non-standard or unparseable frames', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
@@ -367,7 +446,7 @@ describe('AI Activities Unit Tests', () => {
expect(res.embedding).toHaveLength(768)
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
expect(requestBody.input).toHaveLength(16)
expect(requestBody.input).toHaveLength(14)
})
it('should throw non-retryable ApplicationFailure when chunk archive contains no valid frames', async () => {
@@ -642,6 +721,45 @@ describe('AI Database Activities Integration', () => {
delete process.env.EMBEDDING_CHUNK_DURATION
delete process.env.EMBEDDING_CHUNK_OVERLAP
})
it('should return existing embeddings for the asset', async () => {
// Ensure agent exists
await prisma.agent.upsert({
where: { id: user.id },
update: {},
create: {
id: user.id,
teamId: team.id,
type: 'embedding',
enabled: true,
config: { provider: 'openai', model: 'gpt-4' },
},
})
await saveAssetEmbeddingsActivity({
assetId: asset.id,
embeddings: [
{ embedding: Array(1536).fill(0.01), startTime: 0.0, endTime: 60.0 },
{ embedding: Array(768).fill(0.02), startTime: 55.0, endTime: 115.0 },
],
})
const ctx = await getEmbeddingContextActivity({
teamId: team.id,
assetId: asset.id,
})
expect(ctx.existingEmbeddings).toHaveLength(2)
const chunk1 = ctx.existingEmbeddings.find((e) => e.startTime === 0)
expect(chunk1).toBeDefined()
expect(chunk1?.has1536).toBe(true)
expect(chunk1?.has768).toBe(false)
const chunk2 = ctx.existingEmbeddings.find((e) => e.startTime === 55)
expect(chunk2).toBeDefined()
expect(chunk2?.has1536).toBe(false)
expect(chunk2?.has768).toBe(true)
})
})
describe('saveAssetEmbeddingsActivity', () => {
@@ -709,4 +827,52 @@ describe('AI Database Activities Integration', () => {
expect(dbEmbeddings[1].has1536).toBe(false)
})
})
describe('deleteAssetEmbeddingsActivity', () => {
it('should delete specific embeddings by id', async () => {
await saveAssetEmbeddingsActivity({
assetId: asset.id,
embeddings: [
{ embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 },
{ embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 },
],
})
const before = await prisma.assetEmbedding.findMany({
where: { assetId: asset.id },
})
expect(before).toHaveLength(2)
const targetId = before[0].id
await deleteAssetEmbeddingsActivity({
assetId: asset.id,
ids: [targetId],
})
const after = await prisma.assetEmbedding.findMany({
where: { assetId: asset.id },
})
expect(after).toHaveLength(1)
expect(after[0].id).not.toBe(targetId)
})
it('should delete all embeddings for an asset when ids is not provided', async () => {
await saveAssetEmbeddingsActivity({
assetId: asset.id,
embeddings: [
{ embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 },
{ embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 },
],
})
await deleteAssetEmbeddingsActivity({
assetId: asset.id,
})
const after = await prisma.assetEmbedding.findMany({
where: { assetId: asset.id },
})
expect(after).toHaveLength(0)
})
})
})
+89 -3
View File
@@ -250,12 +250,49 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
const embeddingProvider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
const existingEmbeddings = await prisma.$queryRaw<ExistingAssetEmbeddingInfo[]>`
SELECT
id,
start_time as "startTime",
end_time as "endTime",
(embedding IS NOT NULL) as "has1536",
(embedding_768 IS NOT NULL) as "has768"
FROM asset_embeddings
WHERE asset_id = ${params.assetId}
`
return {
agent,
asset,
chunkDuration,
chunkOverlap,
embeddingProvider,
existingEmbeddings,
}
}
export interface ExistingAssetEmbeddingInfo {
id: string
startTime: number | null
endTime: number | null
has1536: boolean
has768: boolean
}
export async function deleteAssetEmbeddingsActivity(params: { assetId: string; ids?: string[] }) {
if (params.ids && params.ids.length > 0) {
await prisma.assetEmbedding.deleteMany({
where: {
assetId: params.assetId,
id: { in: params.ids },
},
})
} else {
await prisma.assetEmbedding.deleteMany({
where: {
assetId: params.assetId,
},
})
}
}
@@ -360,6 +397,38 @@ export async function generateImageEmbeddingActivity(
}
}
/**
* Standard aspect ratios for video embedding:
* 1:1, 4:3, 3:4, 16:9, 9:16, 21:9 (including ultrawide 64:27), and 9:21.
*
* For these standard ratios, 15 frames guarantees that total input tokens stay strictly
* within the 2048 token signature bucket (<= 1982 tokens). For non-standard or custom
* aspect ratios (e.g. 5:3, 1.4:1), 14 frames is used to mathematically guarantee
* that sequence length never exceeds 2048 tokens (<= 1990 tokens), preventing OOM errors.
*/
export function getGemmaEmbeddingTargetFrameCount(width?: number, height?: number): number {
if (!width || !height || width <= 0 || height <= 0) {
return 14
}
const ratio = width / height
const standardRatios = [
1 / 1, // 1:1
4 / 3, // 4:3
3 / 4, // 3:4
16 / 9, // 16:9
9 / 16, // 9:16
21 / 9, // 21:9
64 / 27, // 21:9 ultrawide (2560x1080, 3440x1440)
9 / 21, // 9:21
27 / 64, // 9:21 ultrawide portrait
]
// Allow up to 3% relative tolerance for minor encoder rounding (e.g. 854x480 vs 16:9)
const isStandard = standardRatios.some((std) => Math.abs(ratio - std) / std <= 0.03)
return isStandard ? 15 : 14
}
export interface GenerateVideoChunkEmbeddingParams {
teamId: string
chunkKey: string
@@ -402,10 +471,27 @@ export async function generateVideoChunkEmbeddingActivity(
})
}
let targetFrameCount = 14
try {
const firstFile = filesMap.get(frameFiles[0])
if (firstFile) {
const firstFileBytes = await firstFile.bytes()
if (firstFileBytes.byteLength > 0) {
const meta = await sharp(Buffer.from(firstFileBytes)).metadata()
targetFrameCount = getGemmaEmbeddingTargetFrameCount(meta.width, meta.height)
}
}
} catch {
targetFrameCount = 14
}
let selectedFrames = frameFiles
if (selectedFrames.length > 16) {
const step = (selectedFrames.length - 1) / 15
selectedFrames = Array.from({ length: 16 }, (_, i) => selectedFrames[Math.round(i * step)])
if (selectedFrames.length > targetFrameCount) {
const step = (selectedFrames.length - 1) / (targetFrameCount - 1)
selectedFrames = Array.from(
{ length: targetFrameCount },
(_, i) => selectedFrames[Math.round(i * step)],
)
}
const input: OpenAiEmbeddingItem[] = await Promise.all(
+3
View File
@@ -626,9 +626,12 @@ export {
generateImageEmbeddingActivity,
generateTextEmbeddingActivity,
generateVideoChunkEmbeddingActivity,
saveAssetEmbeddingsActivity,
deleteAssetEmbeddingsActivity,
type ExtractAiMetadataParams,
type GeneratedEmbedding,
type GenerateTextEmbeddingParams,
type ExistingAssetEmbeddingInfo,
} from './activities/ai'
export * from './database-session-storage'
export * from './context/serialize-context'
@@ -38,6 +38,9 @@ describe('Agent Embedding Workflow', () => {
saveAssetEmbeddingsActivity: Object.assign(vi.fn(), {
_activityName: 'saveAssetEmbeddingsActivity',
}),
deleteAssetEmbeddingsActivity: Object.assign(vi.fn(), {
_activityName: 'deleteAssetEmbeddingsActivity',
}),
updateTaskUsageActivity: Object.assign(vi.fn(), { _activityName: 'updateTaskUsageActivity' }),
createCommentActivity: Object.assign(vi.fn(), { _activityName: 'createCommentActivity' }),
updateCommentActivity: Object.assign(vi.fn(), { _activityName: 'updateCommentActivity' }),
@@ -360,14 +363,19 @@ describe('Agent Embedding Workflow', () => {
model: 'gemini-embedding-2',
})
// Verify embeddings saved
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
// Verify embeddings saved per chunk immediately
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(3)
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(1, {
assetId: 'a1',
embeddings: [
{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 },
{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 },
{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 },
],
embeddings: [{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 }],
})
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(2, {
assetId: 'a1',
embeddings: [{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 }],
})
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(3, {
assetId: 'a1',
embeddings: [{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 }],
})
})
@@ -453,6 +461,49 @@ describe('Agent Embedding Workflow', () => {
})
})
it('should save completed chunks to DB immediately before subsequent chunk failure', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'a1',
mediaType: 'video/mp4',
storageKey: { key: 'test.mp4' },
media: {
duration: 150.0,
videoTranscodes: [{ key: 'test-transcoded.mp4' }],
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
// Chunk 1 succeeds, Chunk 2 fails
mockActivities.generateVideoChunkEmbeddingActivity
.mockResolvedValueOnce({
embedding: [0.1, 0.2],
usage: { inputTokens: 3, outputTokens: 3, model: 'gemini-embedding-2' },
})
.mockRejectedValueOnce(new Error('Rate limit exceeded at chunk 2'))
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'a1',
teamId: 't1',
},
})
await expect(agentEmbeddingMedia(task)).rejects.toThrow('Rate limit exceeded at chunk 2')
// Chunk 1 MUST have been saved to the database before chunk 2 failed
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(1)
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
assetId: 'a1',
embeddings: [{ embedding: [0.1, 0.2], startTime: 0, endTime: 60 }],
})
})
it('should not create redundant chunks when video exactly matches chunk duration', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
@@ -529,4 +580,96 @@ describe('Agent Embedding Workflow', () => {
}),
)
})
it('should skip already completed chunks when resuming a video embedding task', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'a1',
mediaType: 'video/mp4',
storageKey: { key: 'test.mp4' },
media: {
duration: 150.0,
videoTranscodes: [{ key: 'test-transcoded.mp4' }],
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
existingEmbeddings: [
{
id: 'emb-1',
startTime: 0,
endTime: 60,
has1536: true,
has768: false,
},
],
})
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'a1',
teamId: 't1',
},
})
await agentEmbeddingMedia(task)
// Chunk 1 (0-60) should be skipped; only chunk 2 (55-115) and chunk 3 (110-150) should be transcoded and embedded
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(2)
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ startTime: 55, endTime: 115 }),
)
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(
2,
expect.objectContaining({ startTime: 110, endTime: 150 }),
)
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(2)
})
it('should purge incompatible embeddings before processing', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'a1',
mediaType: 'image/png',
storageKey: { key: 'test.png' },
},
embeddingProvider: 'gemma-embedding-2-openai',
existingEmbeddings: [
{
id: 'old-gemini-emb',
startTime: null,
endTime: null,
has1536: true,
has768: false,
},
],
})
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'a1',
teamId: 't1',
},
})
await agentEmbeddingMedia(task)
// Should purge incompatible old-gemini-emb
expect(mockActivities.deleteAssetEmbeddingsActivity).toHaveBeenCalledWith({
assetId: 'a1',
ids: ['old-gemini-emb'],
})
// Should generate and save new embedding
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalled()
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalled()
})
})
+84 -27
View File
@@ -13,6 +13,14 @@ export interface GeneratedEmbedding {
endTime?: number
}
export interface ExistingAssetEmbeddingInfo {
id: string
startTime: number | null
endTime: number | null
has1536: boolean
has768: boolean
}
export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
const {
updateTaskStatusActivity,
@@ -20,6 +28,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
generateImageEmbeddingActivity,
generateVideoChunkEmbeddingActivity,
saveAssetEmbeddingsActivity,
deleteAssetEmbeddingsActivity,
updateTaskUsageActivity,
createCommentActivity,
updateCommentActivity,
@@ -62,15 +71,41 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
}
// 1. Fetch Agent Context
const context = await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, {
const context = (await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, {
teamId: task.teamId,
assetId: task.assetId,
})
})) as unknown as {
asset: {
id: string
mediaType: string | null
storageKey?: { key: string } | null
media?: unknown
}
chunkDuration: number
chunkOverlap: number
embeddingProvider: string
existingEmbeddings?: ExistingAssetEmbeddingInfo[]
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const { asset, chunkDuration, chunkOverlap, embeddingProvider } = context as any
const { asset, chunkDuration, chunkOverlap, embeddingProvider, existingEmbeddings } = context
const isGemmaProvider = embeddingProvider === 'gemma-embedding-2-openai'
const existing = existingEmbeddings || []
// Purge incompatible embeddings (e.g. 1536d when Gemma is used, or 768d when Gemini is used)
const incompatibleIds = existing
.filter((e) => (isGemmaProvider ? e.has1536 || !e.has768 : e.has768 || !e.has1536))
.map((e) => e.id)
if (incompatibleIds.length > 0) {
await executeActivity(agentWorkerQueue, deleteAssetEmbeddingsActivity, {
assetId: task.assetId,
ids: incompatibleIds,
})
}
const validExistingChunks = existing.filter((e) => !incompatibleIds.includes(e.id))
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
const is3dWithViews =
mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0
@@ -114,7 +149,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
}
const results: GeneratedEmbedding[] = []
const usage = {
model: isGemmaProvider ? 'gemma-embedding-2-openai' : 'gemini-embedding-2',
inputTokens: 0,
@@ -122,15 +156,23 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
}
if (isImage) {
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
teamId: task.teamId,
assetKey: embeddingKey,
mediaType: embeddingMediaType,
})
results.push({ embedding: result.embedding })
if (result.usage) {
usage.inputTokens += result.usage.inputTokens || 0
usage.outputTokens += result.usage.outputTokens || 0
const alreadyEmbedded = validExistingChunks.some(
(e) => e.startTime === null && e.endTime === null,
)
if (!alreadyEmbedded) {
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
teamId: task.teamId,
assetKey: embeddingKey,
mediaType: embeddingMediaType,
})
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
assetId: task.assetId,
embeddings: [{ embedding: result.embedding }],
})
if (result.usage) {
usage.inputTokens += result.usage.inputTokens || 0
usage.outputTokens += result.usage.outputTokens || 0
}
}
} else if (isVideo) {
// Get video duration
@@ -146,6 +188,23 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
end = duration
}
// Check if this chunk is already completed
const chunkExists = validExistingChunks.some(
(c) =>
c.startTime !== null &&
c.endTime !== null &&
Math.abs(c.startTime - start) < 0.1 &&
Math.abs(c.endTime - end) < 0.1,
)
if (chunkExists) {
// If this chunk reached the end of the video, stop
if (end >= duration) {
break
}
continue
}
// Step A: Slice chunk on transcode worker and upload to S3
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
assetId: asset.id,
@@ -167,10 +226,16 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
},
)
results.push({
embedding: chunkResult.embedding,
startTime: start,
endTime: end,
// Step C: Save chunk embedding to database immediately
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
assetId: task.assetId,
embeddings: [
{
embedding: chunkResult.embedding,
startTime: start,
endTime: end,
},
],
})
if (chunkResult.usage) {
@@ -178,7 +243,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
usage.outputTokens += chunkResult.usage.outputTokens || 0
}
} finally {
// Step C: Delete temporary chunk from S3
// Step D: Delete temporary chunk from S3
await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, {
key: chunkKey,
})
@@ -191,14 +256,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
}
}
// Save computed embeddings
if (results.length > 0) {
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
assetId: task.assetId,
embeddings: results,
})
}
// Update Usage
await executeActivity(agentWorkerQueue, updateTaskUsageActivity, {
taskId: task.id,
+1 -1
View File
@@ -63,7 +63,7 @@ export class TextEmbeddingService {
}
}
const signal = options?.signal ?? AbortSignal.timeout(5000)
const signal = options?.signal ?? AbortSignal.timeout(15000)
let result: TextEmbeddingResult
if (provider === 'gemma-embedding-2-openai') {