mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-19 14:16:29 +00:00
fix(agent): adaptively set video embedding frame count based on aspect ratio (#477)
* fix(agent): adaptively set video embedding frame count based on aspect ratio Adjust sampled video frame counts between 15 and 14 based on the video's aspect ratio. Standard aspect ratios (1:1, 4:3, 3:4, 16:9, 9:16, 21:9) stay within the 2048 token signature bucket when using 15 frames, while non-standard or custom aspect ratios use 14 frames to guarantee that sequence length never exceeds 2048 tokens, preventing out-of-memory errors on the embedding server. * feat(agent): persist video chunk embeddings incrementally and resume on retry (#478)
This commit is contained in:
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Added
|
||||
|
||||
* **agent**: Save video chunk embeddings incrementally to the database as each chunk finishes, preserving completed progress on failure and resuming from where it left off on retry
|
||||
* **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -4,8 +4,10 @@ import {
|
||||
extractAiMetadataActivity,
|
||||
getEmbeddingContextActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
deleteAssetEmbeddingsActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
getGemmaEmbeddingTargetFrameCount,
|
||||
} from './ai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
@@ -335,7 +337,84 @@ describe('AI Activities Unit Tests', () => {
|
||||
expect(mockFetch).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should sample uniform 16 frames when chunk archive contains more than 16 frames', async () => {
|
||||
describe('getGemmaEmbeddingTargetFrameCount', () => {
|
||||
it('should return 15 for standard aspect ratios', () => {
|
||||
// 16:9
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1920, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1280, 720)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(854, 480)).toBe(15)
|
||||
// 9:16
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1080, 1920)).toBe(15)
|
||||
// 1:1
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1080, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(480, 480)).toBe(15)
|
||||
// 4:3
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1440, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(640, 480)).toBe(15)
|
||||
// 3:4
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1080, 1440)).toBe(15)
|
||||
// 21:9
|
||||
expect(getGemmaEmbeddingTargetFrameCount(2560, 1080)).toBe(15)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(3440, 1440)).toBe(15)
|
||||
})
|
||||
|
||||
it('should return 14 for non-standard aspect ratios and missing/invalid dimensions', () => {
|
||||
expect(getGemmaEmbeddingTargetFrameCount(1000, 600)).toBe(14) // 5:3
|
||||
expect(getGemmaEmbeddingTargetFrameCount(700, 500)).toBe(14) // 1.4:1
|
||||
expect(getGemmaEmbeddingTargetFrameCount(854, 672)).toBe(14) // ~1.27:1
|
||||
expect(getGemmaEmbeddingTargetFrameCount(undefined, undefined)).toBe(14)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(0, 0)).toBe(14)
|
||||
expect(getGemmaEmbeddingTargetFrameCount(-100, 100)).toBe(14)
|
||||
})
|
||||
})
|
||||
|
||||
it('should sample uniform 15 frames when chunk archive contains standard 16:9 frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.3) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const sampleJpg = await sharp({
|
||||
create: {
|
||||
width: 160,
|
||||
height: 90,
|
||||
channels: 3,
|
||||
background: { r: 0, g: 0, b: 0 },
|
||||
},
|
||||
})
|
||||
.jpeg()
|
||||
.toBuffer()
|
||||
|
||||
const frames: Record<string, Buffer> = {}
|
||||
for (let i = 1; i <= 30; i++) {
|
||||
frames[`frame_${String(i).padStart(4, '0')}.jpg`] = sampleJpg
|
||||
}
|
||||
const sampleArchive = new Bun.Archive(frames)
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
|
||||
expect(requestBody.input).toHaveLength(15)
|
||||
})
|
||||
|
||||
it('should fallback to 14 frames when chunk archive contains non-standard or unparseable frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
@@ -367,7 +446,7 @@ describe('AI Activities Unit Tests', () => {
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
|
||||
expect(requestBody.input).toHaveLength(16)
|
||||
expect(requestBody.input).toHaveLength(14)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when chunk archive contains no valid frames', async () => {
|
||||
@@ -642,6 +721,45 @@ describe('AI Database Activities Integration', () => {
|
||||
delete process.env.EMBEDDING_CHUNK_DURATION
|
||||
delete process.env.EMBEDDING_CHUNK_OVERLAP
|
||||
})
|
||||
|
||||
it('should return existing embeddings for the asset', async () => {
|
||||
// Ensure agent exists
|
||||
await prisma.agent.upsert({
|
||||
where: { id: user.id },
|
||||
update: {},
|
||||
create: {
|
||||
id: user.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
config: { provider: 'openai', model: 'gpt-4' },
|
||||
},
|
||||
})
|
||||
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings: [
|
||||
{ embedding: Array(1536).fill(0.01), startTime: 0.0, endTime: 60.0 },
|
||||
{ embedding: Array(768).fill(0.02), startTime: 55.0, endTime: 115.0 },
|
||||
],
|
||||
})
|
||||
|
||||
const ctx = await getEmbeddingContextActivity({
|
||||
teamId: team.id,
|
||||
assetId: asset.id,
|
||||
})
|
||||
|
||||
expect(ctx.existingEmbeddings).toHaveLength(2)
|
||||
const chunk1 = ctx.existingEmbeddings.find((e) => e.startTime === 0)
|
||||
expect(chunk1).toBeDefined()
|
||||
expect(chunk1?.has1536).toBe(true)
|
||||
expect(chunk1?.has768).toBe(false)
|
||||
|
||||
const chunk2 = ctx.existingEmbeddings.find((e) => e.startTime === 55)
|
||||
expect(chunk2).toBeDefined()
|
||||
expect(chunk2?.has1536).toBe(false)
|
||||
expect(chunk2?.has768).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('saveAssetEmbeddingsActivity', () => {
|
||||
@@ -709,4 +827,52 @@ describe('AI Database Activities Integration', () => {
|
||||
expect(dbEmbeddings[1].has1536).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('deleteAssetEmbeddingsActivity', () => {
|
||||
it('should delete specific embeddings by id', async () => {
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings: [
|
||||
{ embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 },
|
||||
{ embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 },
|
||||
],
|
||||
})
|
||||
|
||||
const before = await prisma.assetEmbedding.findMany({
|
||||
where: { assetId: asset.id },
|
||||
})
|
||||
expect(before).toHaveLength(2)
|
||||
|
||||
const targetId = before[0].id
|
||||
await deleteAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
ids: [targetId],
|
||||
})
|
||||
|
||||
const after = await prisma.assetEmbedding.findMany({
|
||||
where: { assetId: asset.id },
|
||||
})
|
||||
expect(after).toHaveLength(1)
|
||||
expect(after[0].id).not.toBe(targetId)
|
||||
})
|
||||
|
||||
it('should delete all embeddings for an asset when ids is not provided', async () => {
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings: [
|
||||
{ embedding: Array(1536).fill(0.01), startTime: 0, endTime: 60 },
|
||||
{ embedding: Array(1536).fill(0.02), startTime: 55, endTime: 115 },
|
||||
],
|
||||
})
|
||||
|
||||
await deleteAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
})
|
||||
|
||||
const after = await prisma.assetEmbedding.findMany({
|
||||
where: { assetId: asset.id },
|
||||
})
|
||||
expect(after).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -250,12 +250,49 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
|
||||
const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
|
||||
const embeddingProvider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
|
||||
const existingEmbeddings = await prisma.$queryRaw<ExistingAssetEmbeddingInfo[]>`
|
||||
SELECT
|
||||
id,
|
||||
start_time as "startTime",
|
||||
end_time as "endTime",
|
||||
(embedding IS NOT NULL) as "has1536",
|
||||
(embedding_768 IS NOT NULL) as "has768"
|
||||
FROM asset_embeddings
|
||||
WHERE asset_id = ${params.assetId}
|
||||
`
|
||||
|
||||
return {
|
||||
agent,
|
||||
asset,
|
||||
chunkDuration,
|
||||
chunkOverlap,
|
||||
embeddingProvider,
|
||||
existingEmbeddings,
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExistingAssetEmbeddingInfo {
|
||||
id: string
|
||||
startTime: number | null
|
||||
endTime: number | null
|
||||
has1536: boolean
|
||||
has768: boolean
|
||||
}
|
||||
|
||||
export async function deleteAssetEmbeddingsActivity(params: { assetId: string; ids?: string[] }) {
|
||||
if (params.ids && params.ids.length > 0) {
|
||||
await prisma.assetEmbedding.deleteMany({
|
||||
where: {
|
||||
assetId: params.assetId,
|
||||
id: { in: params.ids },
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await prisma.assetEmbedding.deleteMany({
|
||||
where: {
|
||||
assetId: params.assetId,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,6 +397,38 @@ export async function generateImageEmbeddingActivity(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard aspect ratios for video embedding:
|
||||
* 1:1, 4:3, 3:4, 16:9, 9:16, 21:9 (including ultrawide 64:27), and 9:21.
|
||||
*
|
||||
* For these standard ratios, 15 frames guarantees that total input tokens stay strictly
|
||||
* within the 2048 token signature bucket (<= 1982 tokens). For non-standard or custom
|
||||
* aspect ratios (e.g. 5:3, 1.4:1), 14 frames is used to mathematically guarantee
|
||||
* that sequence length never exceeds 2048 tokens (<= 1990 tokens), preventing OOM errors.
|
||||
*/
|
||||
export function getGemmaEmbeddingTargetFrameCount(width?: number, height?: number): number {
|
||||
if (!width || !height || width <= 0 || height <= 0) {
|
||||
return 14
|
||||
}
|
||||
|
||||
const ratio = width / height
|
||||
const standardRatios = [
|
||||
1 / 1, // 1:1
|
||||
4 / 3, // 4:3
|
||||
3 / 4, // 3:4
|
||||
16 / 9, // 16:9
|
||||
9 / 16, // 9:16
|
||||
21 / 9, // 21:9
|
||||
64 / 27, // 21:9 ultrawide (2560x1080, 3440x1440)
|
||||
9 / 21, // 9:21
|
||||
27 / 64, // 9:21 ultrawide portrait
|
||||
]
|
||||
|
||||
// Allow up to 3% relative tolerance for minor encoder rounding (e.g. 854x480 vs 16:9)
|
||||
const isStandard = standardRatios.some((std) => Math.abs(ratio - std) / std <= 0.03)
|
||||
return isStandard ? 15 : 14
|
||||
}
|
||||
|
||||
export interface GenerateVideoChunkEmbeddingParams {
|
||||
teamId: string
|
||||
chunkKey: string
|
||||
@@ -402,10 +471,27 @@ export async function generateVideoChunkEmbeddingActivity(
|
||||
})
|
||||
}
|
||||
|
||||
let targetFrameCount = 14
|
||||
try {
|
||||
const firstFile = filesMap.get(frameFiles[0])
|
||||
if (firstFile) {
|
||||
const firstFileBytes = await firstFile.bytes()
|
||||
if (firstFileBytes.byteLength > 0) {
|
||||
const meta = await sharp(Buffer.from(firstFileBytes)).metadata()
|
||||
targetFrameCount = getGemmaEmbeddingTargetFrameCount(meta.width, meta.height)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
targetFrameCount = 14
|
||||
}
|
||||
|
||||
let selectedFrames = frameFiles
|
||||
if (selectedFrames.length > 16) {
|
||||
const step = (selectedFrames.length - 1) / 15
|
||||
selectedFrames = Array.from({ length: 16 }, (_, i) => selectedFrames[Math.round(i * step)])
|
||||
if (selectedFrames.length > targetFrameCount) {
|
||||
const step = (selectedFrames.length - 1) / (targetFrameCount - 1)
|
||||
selectedFrames = Array.from(
|
||||
{ length: targetFrameCount },
|
||||
(_, i) => selectedFrames[Math.round(i * step)],
|
||||
)
|
||||
}
|
||||
|
||||
const input: OpenAiEmbeddingItem[] = await Promise.all(
|
||||
|
||||
@@ -626,9 +626,12 @@ export {
|
||||
generateImageEmbeddingActivity,
|
||||
generateTextEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
deleteAssetEmbeddingsActivity,
|
||||
type ExtractAiMetadataParams,
|
||||
type GeneratedEmbedding,
|
||||
type GenerateTextEmbeddingParams,
|
||||
type ExistingAssetEmbeddingInfo,
|
||||
} from './activities/ai'
|
||||
export * from './database-session-storage'
|
||||
export * from './context/serialize-context'
|
||||
|
||||
@@ -38,6 +38,9 @@ describe('Agent Embedding Workflow', () => {
|
||||
saveAssetEmbeddingsActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'saveAssetEmbeddingsActivity',
|
||||
}),
|
||||
deleteAssetEmbeddingsActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'deleteAssetEmbeddingsActivity',
|
||||
}),
|
||||
updateTaskUsageActivity: Object.assign(vi.fn(), { _activityName: 'updateTaskUsageActivity' }),
|
||||
createCommentActivity: Object.assign(vi.fn(), { _activityName: 'createCommentActivity' }),
|
||||
updateCommentActivity: Object.assign(vi.fn(), { _activityName: 'updateCommentActivity' }),
|
||||
@@ -360,14 +363,19 @@ describe('Agent Embedding Workflow', () => {
|
||||
model: 'gemini-embedding-2',
|
||||
})
|
||||
|
||||
// Verify embeddings saved
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
// Verify embeddings saved per chunk immediately
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(3)
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(1, {
|
||||
assetId: 'a1',
|
||||
embeddings: [
|
||||
{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 },
|
||||
{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 },
|
||||
{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 },
|
||||
],
|
||||
embeddings: [{ embedding: [0.3, 0.4], startTime: 0, endTime: 60 }],
|
||||
})
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(2, {
|
||||
assetId: 'a1',
|
||||
embeddings: [{ embedding: [0.3, 0.4], startTime: 55, endTime: 115 }],
|
||||
})
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenNthCalledWith(3, {
|
||||
assetId: 'a1',
|
||||
embeddings: [{ embedding: [0.3, 0.4], startTime: 110, endTime: 150 }],
|
||||
})
|
||||
})
|
||||
|
||||
@@ -453,6 +461,49 @@ describe('Agent Embedding Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should save completed chunks to DB immediately before subsequent chunk failure', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 150.0,
|
||||
videoTranscodes: [{ key: 'test-transcoded.mp4' }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
})
|
||||
|
||||
// Chunk 1 succeeds, Chunk 2 fails
|
||||
mockActivities.generateVideoChunkEmbeddingActivity
|
||||
.mockResolvedValueOnce({
|
||||
embedding: [0.1, 0.2],
|
||||
usage: { inputTokens: 3, outputTokens: 3, model: 'gemini-embedding-2' },
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('Rate limit exceeded at chunk 2'))
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await expect(agentEmbeddingMedia(task)).rejects.toThrow('Rate limit exceeded at chunk 2')
|
||||
|
||||
// Chunk 1 MUST have been saved to the database before chunk 2 failed
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(1)
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
embeddings: [{ embedding: [0.1, 0.2], startTime: 0, endTime: 60 }],
|
||||
})
|
||||
})
|
||||
|
||||
it('should not create redundant chunks when video exactly matches chunk duration', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
@@ -529,4 +580,96 @@ describe('Agent Embedding Workflow', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should skip already completed chunks when resuming a video embedding task', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 150.0,
|
||||
videoTranscodes: [{ key: 'test-transcoded.mp4' }],
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
existingEmbeddings: [
|
||||
{
|
||||
id: 'emb-1',
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
has1536: true,
|
||||
has768: false,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Chunk 1 (0-60) should be skipped; only chunk 2 (55-115) and chunk 3 (110-150) should be transcoded and embedded
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledTimes(2)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ startTime: 55, endTime: 115 }),
|
||||
)
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({ startTime: 110, endTime: 150 }),
|
||||
)
|
||||
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('should purge incompatible embeddings before processing', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'image/png',
|
||||
storageKey: { key: 'test.png' },
|
||||
},
|
||||
embeddingProvider: 'gemma-embedding-2-openai',
|
||||
existingEmbeddings: [
|
||||
{
|
||||
id: 'old-gemini-emb',
|
||||
startTime: null,
|
||||
endTime: null,
|
||||
has1536: true,
|
||||
has768: false,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
// Should purge incompatible old-gemini-emb
|
||||
expect(mockActivities.deleteAssetEmbeddingsActivity).toHaveBeenCalledWith({
|
||||
assetId: 'a1',
|
||||
ids: ['old-gemini-emb'],
|
||||
})
|
||||
|
||||
// Should generate and save new embedding
|
||||
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalled()
|
||||
expect(mockActivities.saveAssetEmbeddingsActivity).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -13,6 +13,14 @@ export interface GeneratedEmbedding {
|
||||
endTime?: number
|
||||
}
|
||||
|
||||
export interface ExistingAssetEmbeddingInfo {
|
||||
id: string
|
||||
startTime: number | null
|
||||
endTime: number | null
|
||||
has1536: boolean
|
||||
has768: boolean
|
||||
}
|
||||
|
||||
export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
const {
|
||||
updateTaskStatusActivity,
|
||||
@@ -20,6 +28,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
saveAssetEmbeddingsActivity,
|
||||
deleteAssetEmbeddingsActivity,
|
||||
updateTaskUsageActivity,
|
||||
createCommentActivity,
|
||||
updateCommentActivity,
|
||||
@@ -62,15 +71,41 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
|
||||
// 1. Fetch Agent Context
|
||||
const context = await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, {
|
||||
const context = (await executeActivity(agentWorkerQueue, getEmbeddingContextActivity, {
|
||||
teamId: task.teamId,
|
||||
assetId: task.assetId,
|
||||
})
|
||||
})) as unknown as {
|
||||
asset: {
|
||||
id: string
|
||||
mediaType: string | null
|
||||
storageKey?: { key: string } | null
|
||||
media?: unknown
|
||||
}
|
||||
chunkDuration: number
|
||||
chunkOverlap: number
|
||||
embeddingProvider: string
|
||||
existingEmbeddings?: ExistingAssetEmbeddingInfo[]
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const { asset, chunkDuration, chunkOverlap, embeddingProvider } = context as any
|
||||
const { asset, chunkDuration, chunkOverlap, embeddingProvider, existingEmbeddings } = context
|
||||
const isGemmaProvider = embeddingProvider === 'gemma-embedding-2-openai'
|
||||
|
||||
const existing = existingEmbeddings || []
|
||||
|
||||
// Purge incompatible embeddings (e.g. 1536d when Gemma is used, or 768d when Gemini is used)
|
||||
const incompatibleIds = existing
|
||||
.filter((e) => (isGemmaProvider ? e.has1536 || !e.has768 : e.has768 || !e.has1536))
|
||||
.map((e) => e.id)
|
||||
|
||||
if (incompatibleIds.length > 0) {
|
||||
await executeActivity(agentWorkerQueue, deleteAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
ids: incompatibleIds,
|
||||
})
|
||||
}
|
||||
|
||||
const validExistingChunks = existing.filter((e) => !incompatibleIds.includes(e.id))
|
||||
|
||||
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
|
||||
const is3dWithViews =
|
||||
mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0
|
||||
@@ -114,7 +149,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
|
||||
}
|
||||
|
||||
const results: GeneratedEmbedding[] = []
|
||||
const usage = {
|
||||
model: isGemmaProvider ? 'gemma-embedding-2-openai' : 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
@@ -122,15 +156,23 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
|
||||
if (isImage) {
|
||||
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
|
||||
teamId: task.teamId,
|
||||
assetKey: embeddingKey,
|
||||
mediaType: embeddingMediaType,
|
||||
})
|
||||
results.push({ embedding: result.embedding })
|
||||
if (result.usage) {
|
||||
usage.inputTokens += result.usage.inputTokens || 0
|
||||
usage.outputTokens += result.usage.outputTokens || 0
|
||||
const alreadyEmbedded = validExistingChunks.some(
|
||||
(e) => e.startTime === null && e.endTime === null,
|
||||
)
|
||||
if (!alreadyEmbedded) {
|
||||
const result = await executeActivity(agentWorkerQueue, generateImageEmbeddingActivity, {
|
||||
teamId: task.teamId,
|
||||
assetKey: embeddingKey,
|
||||
mediaType: embeddingMediaType,
|
||||
})
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: [{ embedding: result.embedding }],
|
||||
})
|
||||
if (result.usage) {
|
||||
usage.inputTokens += result.usage.inputTokens || 0
|
||||
usage.outputTokens += result.usage.outputTokens || 0
|
||||
}
|
||||
}
|
||||
} else if (isVideo) {
|
||||
// Get video duration
|
||||
@@ -146,6 +188,23 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
end = duration
|
||||
}
|
||||
|
||||
// Check if this chunk is already completed
|
||||
const chunkExists = validExistingChunks.some(
|
||||
(c) =>
|
||||
c.startTime !== null &&
|
||||
c.endTime !== null &&
|
||||
Math.abs(c.startTime - start) < 0.1 &&
|
||||
Math.abs(c.endTime - end) < 0.1,
|
||||
)
|
||||
|
||||
if (chunkExists) {
|
||||
// If this chunk reached the end of the video, stop
|
||||
if (end >= duration) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Step A: Slice chunk on transcode worker and upload to S3
|
||||
const chunkRes = await executeActivity(transcodeWorkerQueue, transcodeVideoChunkActivity, {
|
||||
assetId: asset.id,
|
||||
@@ -167,10 +226,16 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
},
|
||||
)
|
||||
|
||||
results.push({
|
||||
embedding: chunkResult.embedding,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
// Step C: Save chunk embedding to database immediately
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: [
|
||||
{
|
||||
embedding: chunkResult.embedding,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
if (chunkResult.usage) {
|
||||
@@ -178,7 +243,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
usage.outputTokens += chunkResult.usage.outputTokens || 0
|
||||
}
|
||||
} finally {
|
||||
// Step C: Delete temporary chunk from S3
|
||||
// Step D: Delete temporary chunk from S3
|
||||
await executeActivity(transcodeWorkerQueue, deleteS3ObjectActivity, {
|
||||
key: chunkKey,
|
||||
})
|
||||
@@ -191,14 +256,6 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
// Save computed embeddings
|
||||
if (results.length > 0) {
|
||||
await executeActivity(agentWorkerQueue, saveAssetEmbeddingsActivity, {
|
||||
assetId: task.assetId,
|
||||
embeddings: results,
|
||||
})
|
||||
}
|
||||
|
||||
// Update Usage
|
||||
await executeActivity(agentWorkerQueue, updateTaskUsageActivity, {
|
||||
taskId: task.id,
|
||||
|
||||
@@ -63,7 +63,7 @@ export class TextEmbeddingService {
|
||||
}
|
||||
}
|
||||
|
||||
const signal = options?.signal ?? AbortSignal.timeout(5000)
|
||||
const signal = options?.signal ?? AbortSignal.timeout(15000)
|
||||
|
||||
let result: TextEmbeddingResult
|
||||
if (provider === 'gemma-embedding-2-openai') {
|
||||
|
||||
Reference in New Issue
Block a user