mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-19 14:16:29 +00:00
feat(agent): support gemma-embedding-2-openai provider (#475)
* feat(agent): support gemma-embedding-2-openai provider Add support for local LiteRT-LM Gemma Embedding 2 provider with 768-dimensional vector embeddings, environment-configured read-only embedding agents, 16-frame uniform sampling for video chunks, and automatic WebP-to-JPEG conversion. * refactor(agent): remove unused legacy generateEmbeddingActivity * chore(agent): align sharp version to 0.35.4 * refactor(agent): rename embedding server env to GEMMA_EMBEDDING_OPENAI_BASE_URL * refactor(agent,transcode): use native Bun.Archive for video chunk frames * fix(transcode,agent): populate buffer entries in Bun.Archive and validate frame sizes
This commit is contained in:
@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Added
|
||||
|
||||
* **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing
|
||||
|
||||
### Fixed
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -19,6 +19,7 @@ async function start() {
|
||||
process.env.BETTER_AUTH_URL = 'http://localhost:5200'
|
||||
process.env.NODE_ENV = 'development'
|
||||
process.env.SHUMAI_E2E = 'true'
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'false'
|
||||
|
||||
// Write variables to .env.e2e file so the teardown can load them
|
||||
const envContent = [`DATABASE_URL=${process.env.DATABASE_URL}`, `S3_BUCKET=${bucketName}`].join(
|
||||
|
||||
@@ -12,6 +12,7 @@ import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { initTranscodeWorkflows } from '@shumai/transcode'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
import { migrateLegacyAgentAvatars } from '@shumai/core/src/agent/migration'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
|
||||
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
|
||||
import { authService } from '@shumai/core/src/auth/auth'
|
||||
@@ -58,6 +59,7 @@ async function run() {
|
||||
// Start services
|
||||
await metadataService.syncSystemFields().catch(console.error)
|
||||
await migrateLegacyAgentAvatars().catch(console.error)
|
||||
await agentService.ensureSystemEmbeddingAgents().catch(console.error)
|
||||
assetService.startCleanupJob()
|
||||
notificationJobService.start()
|
||||
workflowService.start()
|
||||
|
||||
@@ -161,6 +161,7 @@
|
||||
"@temporalio/workflow": "1.23.0",
|
||||
"adm-zip": "0.6.0",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"sharp": "0.35.4",
|
||||
"typebox": "1.3.22",
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.5.4",
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
"@temporalio/workflow": "1.23.0",
|
||||
"adm-zip": "0.6.0",
|
||||
"jittered-fractional-indexing": "1.0.1",
|
||||
"sharp": "0.35.4",
|
||||
"typebox": "1.3.22",
|
||||
"ulid": "3.0.2",
|
||||
"zod": "4.5.4"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import {
|
||||
generateTextEmbeddingActivity,
|
||||
extractAiMetadataActivity,
|
||||
@@ -6,12 +6,12 @@ import {
|
||||
saveAssetEmbeddingsActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
generateEmbeddingActivity,
|
||||
} from './ai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { prisma, AssetType, AssetStatus } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import sharp from 'sharp'
|
||||
|
||||
vi.mock('fs', async (importOriginal) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- importOriginal returns the raw fs module, typed dynamically as any here
|
||||
@@ -54,7 +54,6 @@ vi.mock('child_process', () => {
|
||||
const execFileMock: any = vi.fn((file: any, args: any, cb: any) => {
|
||||
cb(null, '120.0\n', '')
|
||||
})
|
||||
// Setup custom promisify behavior to return { stdout, stderr }
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mock functions dynamically return varying types based on test command calls
|
||||
execFileMock[Symbol.for('nodejs.util.promisify.custom')] = vi.fn(async (file: any) => {
|
||||
if (file === 'ffprobe') {
|
||||
@@ -90,6 +89,14 @@ describe('AI Activities Unit Tests', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
process.env.GEMINI_API_KEY = 'test-key'
|
||||
delete process.env.EMBEDDING_PROVIDER
|
||||
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
delete process.env.EMBEDDING_PROVIDER
|
||||
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
|
||||
})
|
||||
|
||||
it('should generate image embedding', async () => {
|
||||
@@ -126,33 +133,6 @@ describe('AI Activities Unit Tests', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should resolve video input via resolveInput and avoid downloadToFile in generateEmbeddingActivity', async () => {
|
||||
vi.mocked(s3Service.downloadToFile).mockClear()
|
||||
vi.mocked(s3Service.resolveInput).mockClear()
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue(
|
||||
'https://mock-storage/files/asset-123/video.mp4',
|
||||
)
|
||||
|
||||
const res = await generateEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetId: 'asset-123',
|
||||
context: {
|
||||
agent: {},
|
||||
asset: {
|
||||
mediaType: 'video/mp4',
|
||||
name: 'sample.mp4',
|
||||
storageKey: { key: 'files/asset-123/video.mp4' },
|
||||
media: { proxyType: 'video' },
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(s3Service.resolveInput).toHaveBeenCalledWith('shumai', 'files/asset-123/video.mp4')
|
||||
expect(s3Service.downloadToFile).not.toHaveBeenCalled()
|
||||
expect(s3Service.getObject).not.toHaveBeenCalledWith('shumai', 'files/asset-123/video.mp4')
|
||||
expect(res.embeddings.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should generate text embedding successfully', async () => {
|
||||
const res = await generateTextEmbeddingActivity({
|
||||
text: 'Search query',
|
||||
@@ -172,6 +152,279 @@ describe('AI Activities Unit Tests', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate text embedding successfully with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.1) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 15 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const res = await generateTextEmbeddingActivity({
|
||||
text: 'Search query',
|
||||
teamId: 't1',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(res.usage.model).toBe('embeddinggemma-2')
|
||||
expect(res.usage.inputTokens).toBe(15)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input: 'task: search query | text: Search query',
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate image embedding successfully with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.2) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('test-image'),
|
||||
contentType: 'image/png',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateImageEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetKey: 'img.png',
|
||||
mediaType: 'image/png',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(res.usage.model).toBe('embeddinggemma-2')
|
||||
expect(res.usage.inputTokens).toBe(140)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input: [
|
||||
{
|
||||
type: 'image_url',
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url: {
|
||||
url: `data:image/png;base64,${Buffer.from('test-image').toString('base64')}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should respect custom GEMMA_EMBEDDING_OPENAI_BASE_URL with or without /v1 suffix', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.2) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 10 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
// Test with /v1
|
||||
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379/v1'
|
||||
await generateTextEmbeddingActivity({ text: 'test query', teamId: 't1' })
|
||||
expect(mockFetch).toHaveBeenLastCalledWith(
|
||||
'http://custom-host:9379/v1/embeddings',
|
||||
expect.anything(),
|
||||
)
|
||||
|
||||
// Test without /v1
|
||||
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379'
|
||||
await generateTextEmbeddingActivity({ text: 'test query', teamId: 't1' })
|
||||
expect(mockFetch).toHaveBeenLastCalledWith(
|
||||
'http://custom-host:9379/v1/embeddings',
|
||||
expect.anything(),
|
||||
)
|
||||
})
|
||||
|
||||
it('should convert webp image to jpeg with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.2) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const webpBuffer = await sharp({
|
||||
create: { width: 10, height: 10, channels: 3, background: { r: 255, g: 0, b: 0 } },
|
||||
})
|
||||
.webp()
|
||||
.toBuffer()
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: webpBuffer,
|
||||
contentType: 'image/webp',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateImageEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
assetKey: 'img.webp',
|
||||
mediaType: 'image/webp',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://localhost:9379/v1/embeddings',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: expect.stringContaining('data:image/jpeg;base64,'),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should generate video chunk embedding from tar archive with gemma-embedding-2-openai', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.3) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const sampleArchive = new Bun.Archive({
|
||||
'frame_0001.jpg': Buffer.from('frame-1'),
|
||||
})
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
expect(res.usage.model).toBe('embeddinggemma-2')
|
||||
expect(mockFetch).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should sample uniform 16 frames when chunk archive contains more than 16 frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'embeddinggemma-2',
|
||||
data: [{ embedding: Array(768).fill(0.3) }],
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
usage: { prompt_tokens: 140 },
|
||||
}),
|
||||
})
|
||||
vi.stubGlobal('fetch', mockFetch)
|
||||
|
||||
const frames: Record<string, Buffer> = {}
|
||||
for (let i = 1; i <= 30; i++) {
|
||||
frames[`frame_${String(i).padStart(4, '0')}.jpg`] = Buffer.from(`frame-${i}`)
|
||||
}
|
||||
const sampleArchive = new Bun.Archive(frames)
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const res = await generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
})
|
||||
|
||||
expect(res.embedding).toHaveLength(768)
|
||||
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
|
||||
expect(requestBody.input).toHaveLength(16)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when chunk archive contains no valid frames', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const sampleArchive = new Bun.Archive({
|
||||
'info.txt': Buffer.from('not an image'),
|
||||
})
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(
|
||||
generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
}),
|
||||
).rejects.toThrow('No extracted frames found in chunk archive')
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when chunk archive data is corrupted', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('corrupted non-tar bytes'),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(
|
||||
generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
}),
|
||||
).rejects.toThrow('Failed to read chunk archive')
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when an extracted frame is empty (0 bytes)', async () => {
|
||||
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
|
||||
const sampleArchive = new Bun.Archive({
|
||||
'frame_0001.jpg': Buffer.alloc(0),
|
||||
})
|
||||
const archiveBytes = await sampleArchive.bytes()
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
|
||||
contentType: 'application/x-tar',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
await expect(
|
||||
generateVideoChunkEmbeddingActivity({
|
||||
teamId: 't1',
|
||||
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
|
||||
}),
|
||||
).rejects.toThrow('is empty (0 bytes)')
|
||||
})
|
||||
|
||||
describe('extractAiMetadataActivity', () => {
|
||||
it('should return existing images if they are already in S3 storage', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mock returned object cast as any due to partial implementation
|
||||
@@ -423,5 +676,37 @@ describe('AI Database Activities Integration', () => {
|
||||
expect(dbEmbeddings[0].endTime).toBe(60.0)
|
||||
expect(dbEmbeddings[1].startTime).toBeNull()
|
||||
})
|
||||
|
||||
it('should save 768-dim embeddings to embedding_768 column in database', async () => {
|
||||
const embeddings = [
|
||||
{
|
||||
embedding: Array(768).fill(0.01),
|
||||
startTime: 0.0,
|
||||
endTime: 60.0,
|
||||
},
|
||||
{
|
||||
embedding: Array(768).fill(0.02),
|
||||
},
|
||||
]
|
||||
|
||||
await saveAssetEmbeddingsActivity({
|
||||
assetId: asset.id,
|
||||
embeddings,
|
||||
})
|
||||
|
||||
// Query raw embeddings from DB
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- db query output is a raw postgres result array typed as any
|
||||
const dbEmbeddings: any[] = await prisma.$queryRaw`
|
||||
SELECT id, asset_id as "assetId", (embedding_768 IS NOT NULL) as "has768", (embedding IS NOT NULL) as "has1536"
|
||||
FROM asset_embeddings
|
||||
WHERE asset_id = ${asset.id}
|
||||
`
|
||||
|
||||
expect(dbEmbeddings.length).toBe(2)
|
||||
expect(dbEmbeddings[0].has768).toBe(true)
|
||||
expect(dbEmbeddings[0].has1536).toBe(false)
|
||||
expect(dbEmbeddings[1].has768).toBe(true)
|
||||
expect(dbEmbeddings[1].has1536).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
+158
-140
@@ -1,17 +1,11 @@
|
||||
import { GoogleGenAI } from '@google/genai'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { execFile } from 'child_process'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import { promisify } from 'util'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
import { ulid } from 'ulid'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
import sharp from 'sharp'
|
||||
|
||||
export interface Usage {
|
||||
inputTokens: number
|
||||
@@ -25,15 +19,6 @@ export interface GeneratedEmbedding {
|
||||
endTime?: number
|
||||
}
|
||||
|
||||
export interface GenerateEmbeddingParams {
|
||||
teamId: string
|
||||
assetId: string
|
||||
context: {
|
||||
agent: unknown
|
||||
asset: unknown
|
||||
}
|
||||
}
|
||||
|
||||
async function generateMultimodalEmbedding(
|
||||
ai: GoogleGenAI,
|
||||
buffer: Buffer,
|
||||
@@ -62,131 +47,61 @@ async function generateMultimodalEmbedding(
|
||||
return values
|
||||
}
|
||||
|
||||
export async function generateEmbeddingActivity(params: GenerateEmbeddingParams): Promise<{
|
||||
embeddings: GeneratedEmbedding[]
|
||||
interface OpenAiEmbeddingItem {
|
||||
type?: string
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url?: {
|
||||
url: string
|
||||
}
|
||||
}
|
||||
|
||||
async function callGemmaEmbeddingServer(input: string | OpenAiEmbeddingItem[]): Promise<{
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- context properties are fetched as JSON from the DB activity
|
||||
const { asset } = params.context as any
|
||||
const rawBaseUrl = process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL || 'http://localhost:9379/v1'
|
||||
|
||||
if (!asset.mediaType) {
|
||||
throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true })
|
||||
}
|
||||
const normalized = rawBaseUrl.replace(/\/+$/, '')
|
||||
const endpoint = normalized.endsWith('/embeddings')
|
||||
? normalized
|
||||
: normalized.endsWith('/v1')
|
||||
? `${normalized}/embeddings`
|
||||
: `${normalized}/v1/embeddings`
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'embeddinggemma-2',
|
||||
input,
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
throw ApplicationFailure.create({
|
||||
message: 'GEMINI_API_KEY environment variable is not configured',
|
||||
nonRetryable: true,
|
||||
message: `Embedding server returned status ${response.status}: ${errorText}`,
|
||||
nonRetryable: false,
|
||||
})
|
||||
}
|
||||
|
||||
const resolvedApiKey = process.env[apiKey] || apiKey
|
||||
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
|
||||
|
||||
const usage: Usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- response json shape is dynamic from external endpoint
|
||||
const json: any = await response.json()
|
||||
const embedding = json.data?.[0]?.embedding
|
||||
if (!embedding || !Array.isArray(embedding)) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `unsupported media type for embeddings: ${asset.mediaType}`,
|
||||
nonRetryable: true,
|
||||
message: 'Invalid embedding returned from embedding server',
|
||||
nonRetryable: false,
|
||||
})
|
||||
}
|
||||
|
||||
const key = asset.storageKey?.key
|
||||
if (!key) {
|
||||
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
|
||||
}
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const results: GeneratedEmbedding[] = []
|
||||
|
||||
if (isImage) {
|
||||
const { buffer: data } = await s3Service.getObject(bucket, key)
|
||||
const embVec = await generateMultimodalEmbedding(ai, data, asset.mediaType)
|
||||
results.push({ embedding: embVec })
|
||||
} else if (isVideo) {
|
||||
const inputSource = await s3Service.resolveInput(bucket, key)
|
||||
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
|
||||
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v',
|
||||
'error',
|
||||
'-show_entries',
|
||||
'format=duration',
|
||||
'-of',
|
||||
'default=noprint_wrappers=1:nokey=1',
|
||||
inputSource,
|
||||
])
|
||||
const duration = parseFloat(stdout.trim())
|
||||
if (isNaN(duration)) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'failed to parse video duration',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const chunkSize = 60.0
|
||||
for (let start = 0.0; start < duration; start += chunkSize) {
|
||||
let end = start + chunkSize
|
||||
if (end > duration) end = duration
|
||||
|
||||
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'warning',
|
||||
...(isRemote
|
||||
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
|
||||
: []),
|
||||
'-ss',
|
||||
start.toString(),
|
||||
'-i',
|
||||
inputSource,
|
||||
'-map',
|
||||
'0:V:0',
|
||||
'-map',
|
||||
'0:a:0?',
|
||||
'-t',
|
||||
(end - start).toString(),
|
||||
'-c',
|
||||
'copy',
|
||||
chunkTmp,
|
||||
])
|
||||
|
||||
const chunkData = fs.readFileSync(chunkTmp)
|
||||
|
||||
try {
|
||||
const embVec = await generateMultimodalEmbedding(ai, chunkData, 'video/mp4')
|
||||
results.push({ embedding: embVec, startTime: start, endTime: end })
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} catch (e: any) {
|
||||
console.error(
|
||||
`failed to generate video embedding for chunk ${start}-${end}: ${e.message}`,
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
if (fs.existsSync(chunkTmp)) {
|
||||
fs.unlinkSync(chunkTmp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeddings: results,
|
||||
usage,
|
||||
embedding,
|
||||
usage: {
|
||||
model: json.model || 'gemma-embedding-2-openai',
|
||||
inputTokens: json.usage?.prompt_tokens ?? 0,
|
||||
outputTokens: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,6 +114,11 @@ export async function generateTextEmbeddingActivity(params: GenerateTextEmbeddin
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
if (provider === 'gemma-embedding-2-openai') {
|
||||
return callGemmaEmbeddingServer(`task: search query | text: ${params.text}`)
|
||||
}
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
@@ -339,12 +259,14 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
|
||||
|
||||
const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '')
|
||||
const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
|
||||
const embeddingProvider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
|
||||
return {
|
||||
agent,
|
||||
asset,
|
||||
chunkDuration,
|
||||
chunkOverlap,
|
||||
embeddingProvider,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -358,16 +280,32 @@ export async function saveAssetEmbeddingsActivity(params: {
|
||||
}) {
|
||||
for (const item of params.embeddings) {
|
||||
const embVec = JSON.stringify(item.embedding)
|
||||
const is768 = item.embedding.length === 768
|
||||
|
||||
if (item.startTime !== undefined && item.endTime !== undefined) {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
|
||||
`
|
||||
if (is768) {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
|
||||
`
|
||||
} else {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
|
||||
`
|
||||
}
|
||||
} else {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
|
||||
`
|
||||
if (is768) {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
|
||||
`
|
||||
} else {
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at)
|
||||
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
|
||||
`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -384,6 +322,30 @@ export async function generateImageEmbeddingActivity(
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.assetKey,
|
||||
)
|
||||
|
||||
if (provider === 'gemma-embedding-2-openai') {
|
||||
let imgData = data
|
||||
let mimeType = params.mediaType || 'image/jpeg'
|
||||
if (mimeType !== 'image/jpeg' && mimeType !== 'image/png') {
|
||||
imgData = await sharp(data).jpeg().toBuffer()
|
||||
mimeType = 'image/jpeg'
|
||||
}
|
||||
return callGemmaEmbeddingServer([
|
||||
{
|
||||
type: 'image_url',
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url: {
|
||||
url: `data:${mimeType};base64,${imgData.toString('base64')}`,
|
||||
},
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
@@ -401,10 +363,6 @@ export async function generateImageEmbeddingActivity(
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.assetKey,
|
||||
)
|
||||
const embVec = await generateMultimodalEmbedding(ai, data, params.mediaType)
|
||||
|
||||
return {
|
||||
@@ -424,6 +382,66 @@ export async function generateVideoChunkEmbeddingActivity(
|
||||
embedding: number[]
|
||||
usage: Usage
|
||||
}> {
|
||||
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
|
||||
const isGemma = provider === 'gemma-embedding-2-openai' || params.chunkKey.endsWith('.tar')
|
||||
|
||||
if (isGemma) {
|
||||
const { buffer: data } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
params.chunkKey,
|
||||
)
|
||||
|
||||
let filesMap: Map<string, File>
|
||||
try {
|
||||
const archive = new Bun.Archive(data)
|
||||
filesMap = await archive.files()
|
||||
} catch (err) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `Failed to read chunk archive ${params.chunkKey}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const frameFiles = Array.from(filesMap.keys())
|
||||
.filter((f) => /\.(jpe?g|png|webp)$/i.test(f))
|
||||
.sort()
|
||||
|
||||
if (frameFiles.length === 0) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `No extracted frames found in chunk archive ${params.chunkKey}`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
let selectedFrames = frameFiles
|
||||
if (selectedFrames.length > 16) {
|
||||
const step = (selectedFrames.length - 1) / 15
|
||||
selectedFrames = Array.from({ length: 16 }, (_, i) => selectedFrames[Math.round(i * step)])
|
||||
}
|
||||
|
||||
const input: OpenAiEmbeddingItem[] = await Promise.all(
|
||||
selectedFrames.map(async (f) => {
|
||||
const file = filesMap.get(f)!
|
||||
const fileBytes = await file.bytes()
|
||||
if (fileBytes.byteLength === 0) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `Extracted frame ${f} in chunk archive ${params.chunkKey} is empty (0 bytes)`,
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
return {
|
||||
type: 'image_url',
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
image_url: {
|
||||
url: `data:image/jpeg;base64,${Buffer.from(fileBytes).toString('base64')}`,
|
||||
},
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
return await callGemmaEmbeddingServer(input)
|
||||
}
|
||||
|
||||
const apiKey = process.env.GEMINI_API_KEY
|
||||
if (!apiKey) {
|
||||
throw ApplicationFailure.create({
|
||||
|
||||
@@ -625,13 +625,11 @@ export function initAgentWorkflows() {
|
||||
export * from './activities/agent'
|
||||
export {
|
||||
extractAiMetadataActivity,
|
||||
generateEmbeddingActivity,
|
||||
generateImageEmbeddingActivity,
|
||||
generateTextEmbeddingActivity,
|
||||
generateVideoChunkEmbeddingActivity,
|
||||
type ExtractAiMetadataParams,
|
||||
type GeneratedEmbedding,
|
||||
type GenerateEmbeddingParams,
|
||||
type GenerateTextEmbeddingParams,
|
||||
} from './activities/ai'
|
||||
export * from './database-session-storage'
|
||||
|
||||
@@ -314,18 +314,21 @@ describe('Agent Embedding Workflow', () => {
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
format: 'video',
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
|
||||
assetId: 'a1',
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
startTime: 55,
|
||||
endTime: 115,
|
||||
format: 'video',
|
||||
})
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
|
||||
assetId: 'a1',
|
||||
assetKey: 'test-transcoded.mp4',
|
||||
startTime: 110,
|
||||
endTime: 150,
|
||||
format: 'video',
|
||||
})
|
||||
|
||||
// Verify video chunk embedding generation (3 chunks) on agent queue
|
||||
@@ -485,4 +488,45 @@ describe('Agent Embedding Workflow', () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should run agent embedding workflow for video with gemma-embedding-2-openai requesting frames format', async () => {
|
||||
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
|
||||
agent: { id: 'b1' },
|
||||
asset: {
|
||||
id: 'a1',
|
||||
mediaType: 'video/mp4',
|
||||
storageKey: { key: 'test.mp4' },
|
||||
media: {
|
||||
duration: 30.0,
|
||||
},
|
||||
},
|
||||
chunkDuration: 60.0,
|
||||
chunkOverlap: 5.0,
|
||||
embeddingProvider: 'gemma-embedding-2-openai',
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'ai_embedding',
|
||||
status: 'pending',
|
||||
assetId: 'a1',
|
||||
teamId: 't1',
|
||||
},
|
||||
})
|
||||
|
||||
await agentEmbeddingMedia(task)
|
||||
|
||||
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
startTime: 0,
|
||||
endTime: 30,
|
||||
format: 'frames',
|
||||
}),
|
||||
)
|
||||
expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
model: 'gemma-embedding-2-openai',
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -68,7 +68,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const { asset, chunkDuration, chunkOverlap } = context as any
|
||||
const { asset, chunkDuration, chunkOverlap, embeddingProvider } = context as any
|
||||
const isGemmaProvider = embeddingProvider === 'gemma-embedding-2-openai'
|
||||
|
||||
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
|
||||
const is3dWithViews =
|
||||
@@ -115,7 +116,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
|
||||
const results: GeneratedEmbedding[] = []
|
||||
const usage = {
|
||||
model: 'gemini-embedding-2',
|
||||
model: isGemmaProvider ? 'gemma-embedding-2-openai' : 'gemini-embedding-2',
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
@@ -151,6 +152,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
|
||||
assetKey: embeddingKey,
|
||||
startTime: start,
|
||||
endTime: end,
|
||||
format: isGemmaProvider ? 'frames' : 'video',
|
||||
})
|
||||
const { chunkKey } = chunkRes
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { authzService } from '@shumai/core/src/authz/authz'
|
||||
@@ -210,6 +210,30 @@ describe('Agent API', () => {
|
||||
itemId: 'agent1',
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 403 when updating embedding agent and ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
|
||||
id: 'agent1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
|
||||
const res = await app.request('/agents/agent1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Updated Embedding Agent',
|
||||
type: 'embedding',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('DELETE /agents/:agentId', () => {
|
||||
@@ -227,6 +251,25 @@ describe('Agent API', () => {
|
||||
itemId: 'agent1',
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 403 when deleting embedding agent and ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
|
||||
id: 'agent1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
|
||||
const res = await app.request('/agents/agent1', {
|
||||
method: 'DELETE',
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('PATCH /agents/:agentId/permission', () => {
|
||||
@@ -271,6 +314,27 @@ describe('Agent API', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('returns 403 when modifying embedding agent permission and ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
|
||||
id: 'agent1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
|
||||
const res = await app.request('/agents/agent1/permission', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ permission: 'owner' }),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
it('denies access when user is not admin', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockRejectedValue(
|
||||
new HTTPException(403, { message: 'Forbidden' }),
|
||||
@@ -410,4 +474,57 @@ describe('Agent API', () => {
|
||||
expect(agentService.listProjectChatAgents).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('read-only embedding agent when ENABLE_EMBEDDING_AGENT is true', () => {
|
||||
beforeEach(() => {
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
vi.mocked(agentService.getAgent).mockResolvedValue({
|
||||
id: 'emb1',
|
||||
teamId: 'team1',
|
||||
type: 'embedding',
|
||||
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
})
|
||||
|
||||
it('rejects PUT /agents/:agentId with 403', async () => {
|
||||
const res = await app.request('/agents/emb1', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: 'Updated Embedding',
|
||||
type: 'embedding',
|
||||
enabled: false,
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.updateAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects DELETE /agents/:agentId with 403', async () => {
|
||||
const res = await app.request('/agents/emb1', {
|
||||
method: 'DELETE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.deleteAgent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects PATCH /agents/:agentId/permission with 403', async () => {
|
||||
const res = await app.request('/agents/emb1/permission', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
permission: 'owner',
|
||||
}),
|
||||
})
|
||||
|
||||
expect(res.status).toBe(403)
|
||||
expect(agentService.updateAgentPermission).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Hono } from 'hono'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { zValidator } from '@hono/zod-validator'
|
||||
import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
|
||||
@@ -92,6 +93,13 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: agentId,
|
||||
})
|
||||
|
||||
const existingAgent = await agentService.getAgent({ agentId })
|
||||
if (existingAgent?.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true') {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be modified',
|
||||
})
|
||||
}
|
||||
|
||||
const agent = await agentService.updateAgent({
|
||||
agentId,
|
||||
...req,
|
||||
@@ -123,6 +131,13 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
id: agentId,
|
||||
})
|
||||
|
||||
const existingAgent = await agentService.getAgent({ agentId })
|
||||
if (existingAgent?.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true') {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be modified',
|
||||
})
|
||||
}
|
||||
|
||||
const agent = await agentService.updateAgentPermission(agentId, req.permission)
|
||||
|
||||
await auditLogService.logAction({
|
||||
@@ -149,6 +164,11 @@ const route = new Hono<{ Variables: { user: User } }>()
|
||||
})
|
||||
|
||||
const existingAgent = await agentService.getAgent({ agentId })
|
||||
if (existingAgent?.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true') {
|
||||
throw new HTTPException(403, {
|
||||
message: 'Embedding agent is managed by environment configuration and cannot be deleted',
|
||||
})
|
||||
}
|
||||
|
||||
await agentService.deleteAgent({
|
||||
agentId,
|
||||
|
||||
@@ -976,4 +976,85 @@ describe('AgentService', () => {
|
||||
expect(ids).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('ensureEmbeddingAgent', () => {
|
||||
test('creates embedding agent when ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team } = await setupTestData(db)
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
|
||||
})
|
||||
expect(agent).toBeDefined()
|
||||
const info = await svc.toAgentInfo(agent!)
|
||||
expect(info.name).toBe('embedding-agent')
|
||||
expect(info.type).toBe('embedding')
|
||||
expect(info.enabled).toBe(true)
|
||||
expect(info.avatarPreset).toBe('avatar-1')
|
||||
expect(info.avatar).toContain('avatar-1.webp')
|
||||
expect(info.readOnly).toBe(true)
|
||||
|
||||
// Ensure idempotency
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const count = await db.agent.count({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
})
|
||||
expect(count).toBe(1)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
test('enables existing embedding agent when ENABLE_EMBEDDING_AGENT is true', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const { team } = await setupTestData(db)
|
||||
|
||||
const bot = await db.user.create({
|
||||
data: { name: 'Existing Embedding', email: 'existing-emb@shumai.ai', type: 'agent' },
|
||||
})
|
||||
await db.agent.create({
|
||||
data: {
|
||||
id: bot.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: false,
|
||||
config: { provider: '', model: '' },
|
||||
},
|
||||
})
|
||||
|
||||
process.env.ENABLE_EMBEDDING_AGENT = 'true'
|
||||
try {
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
|
||||
})
|
||||
expect(agent?.id).toBe(bot.id)
|
||||
expect(agent?.enabled).toBe(true)
|
||||
const info = await svc.toAgentInfo(agent!)
|
||||
expect(info.readOnly).toBe(true)
|
||||
} finally {
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
}
|
||||
})
|
||||
|
||||
test('does nothing when ENABLE_EMBEDDING_AGENT is false or not set', async () => {
|
||||
const db = prisma
|
||||
const svc = new AgentService()
|
||||
const team = await db.team.create({ data: { name: 'Team No Agent' } })
|
||||
|
||||
delete process.env.ENABLE_EMBEDDING_AGENT
|
||||
await svc.ensureEmbeddingAgentForTeam(team.id)
|
||||
const agent = await db.agent.findFirst({
|
||||
where: { teamId: team.id, type: 'embedding' },
|
||||
})
|
||||
expect(agent).toBeNull()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -43,8 +43,8 @@ interface AgentInfoSource {
|
||||
config: unknown
|
||||
soul: string | null
|
||||
user: { name: string; image: string | null }
|
||||
skills: Array<{ id: string; skillId: string; skill: { id: string; name: string } }>
|
||||
mcpServers: Array<{ mcpServerId: string }>
|
||||
skills?: Array<{ id: string; skillId: string; skill: { id: string; name: string } }>
|
||||
mcpServers?: Array<{ mcpServerId: string }>
|
||||
}
|
||||
|
||||
function extractS3Key(urlOrKey?: string | null): string | null | undefined {
|
||||
@@ -462,6 +462,8 @@ export class AgentService {
|
||||
*/
|
||||
async toAgentInfo(agent: AgentInfoSource): Promise<AgentInfo> {
|
||||
const config = agent.config as unknown as PrismaJson.AgentConfig
|
||||
const isReadOnly = agent.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true'
|
||||
|
||||
return {
|
||||
id: agent.id,
|
||||
name: agent.user.name,
|
||||
@@ -475,13 +477,14 @@ export class AgentService {
|
||||
thinkingLevel: config.thinkingLevel || 'off',
|
||||
systemPrompt: config.systemPrompt,
|
||||
soul: agent.soul || undefined,
|
||||
skills: agent.skills.map((s) => ({
|
||||
skills: (agent.skills || []).map((s) => ({
|
||||
id: s.id,
|
||||
skillId: s.skillId,
|
||||
skill: s.skill,
|
||||
})),
|
||||
mcpServerIds: (agent.mcpServers || []).map((m) => m.mcpServerId),
|
||||
deniedTools: config.deniedTools || [],
|
||||
readOnly: isReadOnly,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,6 +721,49 @@ export class AgentService {
|
||||
params,
|
||||
)
|
||||
}
|
||||
|
||||
async ensureSystemEmbeddingAgents(): Promise<void> {
|
||||
if (process.env.ENABLE_EMBEDDING_AGENT !== 'true') {
|
||||
return
|
||||
}
|
||||
|
||||
const teams = await this.prismaClient.team.findMany({
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
for (const team of teams) {
|
||||
await this.ensureEmbeddingAgentForTeam(team.id)
|
||||
}
|
||||
}
|
||||
|
||||
async ensureEmbeddingAgentForTeam(teamId: string): Promise<void> {
|
||||
if (process.env.ENABLE_EMBEDDING_AGENT !== 'true') {
|
||||
return
|
||||
}
|
||||
|
||||
const existing = await this.prismaClient.agent.findFirst({
|
||||
where: {
|
||||
teamId,
|
||||
type: 'embedding',
|
||||
},
|
||||
})
|
||||
|
||||
if (!existing) {
|
||||
await this.createAgent({
|
||||
teamId,
|
||||
name: 'embedding-agent',
|
||||
type: 'embedding',
|
||||
avatar: 'avatar-1',
|
||||
thinkingLevel: 'off',
|
||||
enabled: true,
|
||||
})
|
||||
} else if (!existing.enabled) {
|
||||
await this.prismaClient.agent.update({
|
||||
where: { id: existing.id },
|
||||
data: { enabled: true },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const agentService = new AgentService()
|
||||
|
||||
@@ -857,6 +857,154 @@ describe('SearchService', () => {
|
||||
|
||||
mockExecuteWait.mockRestore()
|
||||
})
|
||||
|
||||
it('matches and sorts 768-dimensional embeddings using embedding_768 column', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'semantic-user-768', email: 'semantic-user-768@example.com', type: 'human' },
|
||||
})
|
||||
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'semantic-team-768' },
|
||||
})
|
||||
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'semantic-project-768', teamId: team.id },
|
||||
})
|
||||
|
||||
const botUser = await prisma.user.create({
|
||||
data: { name: 'Embedding Bot 768', email: 'emb-768@example.com', type: 'agent' },
|
||||
})
|
||||
const provider = await prisma.provider.create({
|
||||
data: {
|
||||
name: 'gemma',
|
||||
teamId: team.id,
|
||||
config: { api: 'openai', apiKey: 'key' } as unknown as PrismaJson.ProviderConfig,
|
||||
},
|
||||
})
|
||||
const model = await prisma.model.create({
|
||||
data: {
|
||||
modelId: 'embeddinggemma-2',
|
||||
name: 'EmbeddingGemma 2',
|
||||
providerId: provider.id,
|
||||
config: {
|
||||
reasoning: false,
|
||||
input: ['text'],
|
||||
contextWindow: 1000,
|
||||
maxTokens: 1000,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
} as unknown as PrismaJson.ModelConfig,
|
||||
},
|
||||
})
|
||||
await prisma.agent.create({
|
||||
data: {
|
||||
id: botUser.id,
|
||||
teamId: team.id,
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
providerId: provider.id,
|
||||
modelId: model.id,
|
||||
config: { provider: 'gemma-embedding-2-openai', model: 'embeddinggemma-2' },
|
||||
},
|
||||
})
|
||||
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root-768',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const videoAssetA = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'VideoA_768.mp4',
|
||||
type: AssetType.file,
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const videoAssetB = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'VideoB_768.mp4',
|
||||
type: AssetType.file,
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
creatorId: user.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
})
|
||||
|
||||
const arrA1 = Array(768).fill(0)
|
||||
arrA1[0] = 1.0
|
||||
const vectorA1 = JSON.stringify(arrA1)
|
||||
|
||||
const arrB1 = Array(768).fill(0)
|
||||
arrB1[0] = 1.0
|
||||
arrB1[1] = 1.0
|
||||
const vectorB1 = JSON.stringify(arrB1)
|
||||
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, created_at, updated_at)
|
||||
VALUES (
|
||||
'chunk-768-a1',
|
||||
${videoAssetA.id},
|
||||
${vectorA1}::vector,
|
||||
0.0,
|
||||
10.0,
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
`
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, created_at, updated_at)
|
||||
VALUES (
|
||||
'chunk-768-b1',
|
||||
${videoAssetB.id},
|
||||
${vectorB1}::vector,
|
||||
5.0,
|
||||
15.0,
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
`
|
||||
|
||||
// Query vector with 768 dims pointing along dimension 0
|
||||
const queryVector = Array(768).fill(0)
|
||||
queryVector[0] = 1.0
|
||||
const mockExecuteWait = vi
|
||||
.spyOn(workflowService, 'executeWait')
|
||||
.mockImplementation(async (task) => {
|
||||
return {
|
||||
...task,
|
||||
status: 'completed',
|
||||
output: {
|
||||
embedding: queryVector,
|
||||
},
|
||||
} as unknown as typeof task
|
||||
})
|
||||
|
||||
const result = await searchService.search(rootFolder.id, {
|
||||
recursively: true,
|
||||
assetType: 'file',
|
||||
operator: 'AND',
|
||||
conditions: [],
|
||||
query: 'find 768 dim match',
|
||||
isSemantic: true,
|
||||
})
|
||||
|
||||
expect(result.data.length).toBeGreaterThanOrEqual(2)
|
||||
expect(result.data[0].id).toBe(videoAssetA.id)
|
||||
expect(result.data[1].id).toBe(videoAssetB.id)
|
||||
|
||||
mockExecuteWait.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
describe('totalSize calculation', () => {
|
||||
|
||||
@@ -68,12 +68,21 @@ export class SearchService {
|
||||
|
||||
// 3. Construct raw SQL using SqlQueryBuilder
|
||||
const vectorJson = JSON.stringify(queryVector)
|
||||
const is768 = queryVector.length === 768
|
||||
const distanceExpr = is768
|
||||
? Prisma.sql`(ae.embedding_768 <=> ${vectorJson}::vector)`
|
||||
: Prisma.sql`(ae.embedding <=> ${vectorJson}::vector)`
|
||||
const notNullCond = is768
|
||||
? Prisma.sql`ae.embedding_768 IS NOT NULL`
|
||||
: Prisma.sql`ae.embedding IS NOT NULL`
|
||||
|
||||
const builder = new SqlQueryBuilder()
|
||||
.select(
|
||||
Prisma.sql`a.id as "assetId", ae.start_time as "startTime", ae.end_time as "endTime", (ae.embedding <=> ${vectorJson}::vector) as "distance"`,
|
||||
Prisma.sql`a.id as "assetId", ae.start_time as "startTime", ae.end_time as "endTime", ${distanceExpr} as "distance"`,
|
||||
)
|
||||
.from(Prisma.sql`assets a JOIN asset_embeddings ae ON a.id = ae.asset_id`)
|
||||
.addWhere(Prisma.sql`a.is_deleted = false`)
|
||||
.addWhere(notNullCond)
|
||||
|
||||
if (targetFolderIds.length > 0) {
|
||||
builder.addWhere(Prisma.sql`a.parent_id = ANY(${targetFolderIds})`)
|
||||
@@ -158,6 +167,7 @@ export class SearchService {
|
||||
.select(Prisma.sql`COUNT(*)`)
|
||||
.from(Prisma.sql`assets a JOIN asset_embeddings ae ON a.id = ae.asset_id`)
|
||||
.addWhere(Prisma.sql`a.is_deleted = false`)
|
||||
.addWhere(notNullCond)
|
||||
|
||||
if (targetFolderIds.length > 0) {
|
||||
countBuilder.addWhere(Prisma.sql`a.parent_id = ANY(${targetFolderIds})`)
|
||||
|
||||
@@ -6,6 +6,7 @@ import { notificationService } from '@shumai/core/src/notification/notification'
|
||||
import { getAvatarUrl } from '@shumai/core/src/user/avatar'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { getAllowedAgentRoles } from '@shumai/core/src/agent/permissions'
|
||||
import { AgentService, agentService } from '@shumai/core/src/agent/agent'
|
||||
import { sandboxService } from '@shumai/core/src/sandbox/sandbox-service'
|
||||
import { turntableService } from '@shumai/core/src/turntable/turntable'
|
||||
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
||||
@@ -46,6 +47,7 @@ export class TeamService {
|
||||
},
|
||||
})
|
||||
await providerService.initBuiltinProviders(team.id)
|
||||
await agentService.ensureEmbeddingAgentForTeam(team.id)
|
||||
}
|
||||
|
||||
return team
|
||||
@@ -86,10 +88,12 @@ export class TeamService {
|
||||
},
|
||||
})
|
||||
|
||||
// We use 'any' here because ProviderService expects the full PrismaClient,
|
||||
// We use 'any' here because ProviderService/AgentService expect the full PrismaClient,
|
||||
// but the TransactionClient is compatible with the methods we use.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new ProviderService(tx as any).initBuiltinProviders(team.id)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new AgentService(tx as any).ensureEmbeddingAgentForTeam(team.id)
|
||||
|
||||
return this.toTeamInfo(updatedTeam)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "asset_embeddings" ADD COLUMN "embedding_768" vector(768);
|
||||
|
||||
-- CreateIndex: HNSW index on asset_embeddings.embedding_768
|
||||
CREATE INDEX "AssetEmbedding_embedding_768_idx"
|
||||
ON "asset_embeddings"
|
||||
USING hnsw (embedding_768 vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 64);
|
||||
@@ -706,17 +706,19 @@ model AssetCommentReaction {
|
||||
}
|
||||
|
||||
model AssetEmbedding {
|
||||
id String @id @default(ulid())
|
||||
embedding Unsupported("vector(1536)")?
|
||||
startTime Float? @map("start_time")
|
||||
endTime Float? @map("end_time")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
id String @id @default(ulid())
|
||||
embedding Unsupported("vector(1536)")?
|
||||
embedding768 Unsupported("vector(768)")? @map("embedding_768")
|
||||
startTime Float? @map("start_time")
|
||||
endTime Float? @map("end_time")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
assetId String @map("asset_id")
|
||||
asset Asset @relation("AssetToEmbedding", fields: [assetId], references: [id], onDelete: Cascade)
|
||||
assetId String @map("asset_id")
|
||||
asset Asset @relation("AssetToEmbedding", fields: [assetId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([embedding], name: "AssetEmbedding_embedding_idx")
|
||||
@@index([embedding768], name: "AssetEmbedding_embedding_768_idx")
|
||||
@@map("asset_embeddings")
|
||||
}
|
||||
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -42,6 +42,7 @@ export const agentInfoSchema = z.object({
|
||||
skills: z.array(agentSkillSchema).optional(),
|
||||
mcpServerIds: z.array(z.string()).optional(),
|
||||
deniedTools: z.array(z.string()).optional(),
|
||||
readOnly: z.boolean().optional(),
|
||||
})
|
||||
export type AgentInfo = z.infer<typeof agentInfoSchema>
|
||||
|
||||
|
||||
@@ -1230,6 +1230,100 @@ describe('Transcode Activities', () => {
|
||||
expect(res.chunkKey).toMatch(/^files\/asset-abc\/tmp-embedding-chunks\/chunk-10-25-.*\.mp4$/)
|
||||
})
|
||||
|
||||
it('should extract frames archive when format is frames', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
_file: string,
|
||||
args: string[],
|
||||
cb: (err: Error | null, result: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
if (_file === 'ffmpeg' && args) {
|
||||
const outPattern = args[args.length - 1]
|
||||
if (outPattern && outPattern.includes('frame_%04d.jpg')) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const fsActual = require('fs')
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const pathActual = require('path')
|
||||
const dir = pathActual.dirname(outPattern)
|
||||
fsActual.writeFileSync(
|
||||
pathActual.join(dir, 'frame_0001.jpg'),
|
||||
Buffer.from('mock-frame'),
|
||||
)
|
||||
}
|
||||
}
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
},
|
||||
)
|
||||
|
||||
const res = await transcodeVideoChunkActivity({
|
||||
assetId: 'asset-abc',
|
||||
filePath: '/tmp/full-video.mp4',
|
||||
startTime: 10,
|
||||
endTime: 25,
|
||||
format: 'frames',
|
||||
})
|
||||
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'ffmpeg',
|
||||
expect.arrayContaining(['-vf', 'fps=1', '-q:v', '2']),
|
||||
expect.any(Function),
|
||||
)
|
||||
expect(child_process.execFile).not.toHaveBeenCalledWith(
|
||||
'tar',
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
)
|
||||
expect(s3Service.putObject).toHaveBeenCalledWith(
|
||||
'shumai',
|
||||
expect.stringMatching(/^files\/asset-abc\/tmp-embedding-chunks\/frames-10-25-.*\.tar$/),
|
||||
expect.any(Buffer),
|
||||
expect.any(Number),
|
||||
'application/x-tar',
|
||||
)
|
||||
expect(res.chunkKey).toMatch(/^files\/asset-abc\/tmp-embedding-chunks\/frames-10-25-.*\.tar$/)
|
||||
|
||||
const putCall = vi.mocked(s3Service.putObject).mock.calls[0]
|
||||
const uploadedBuffer = putCall[2] as Buffer
|
||||
const uploadedArchive = new Bun.Archive(uploadedBuffer)
|
||||
const archiveFiles = await uploadedArchive.files()
|
||||
expect(archiveFiles.has('frame_0001.jpg')).toBe(true)
|
||||
const uploadedFile = archiveFiles.get('frame_0001.jpg')!
|
||||
expect(uploadedFile.size).toBeGreaterThan(0)
|
||||
const uploadedFileBytes = await uploadedFile.bytes()
|
||||
expect(uploadedFileBytes.byteLength).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should sample uniform 16 frames when chunk duration > 16s', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- child_process.execFile mockImplementation returns void rather than ChildProcess in unit tests
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
_cmd: unknown,
|
||||
_args: unknown,
|
||||
cb: (err: null, res: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
cb(null, {
|
||||
stdout: '',
|
||||
stderr: '',
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
await transcodeVideoChunkActivity({
|
||||
assetId: 'asset-abc',
|
||||
filePath: '/tmp/full-video.mp4',
|
||||
startTime: 0,
|
||||
endTime: 60,
|
||||
format: 'frames',
|
||||
})
|
||||
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'ffmpeg',
|
||||
expect.arrayContaining(['-vf', `fps=${16 / 60}`, '-vframes', '16', '-q:v', '2']),
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when neither filePath nor assetKey is provided', async () => {
|
||||
expect.assertions(2)
|
||||
try {
|
||||
|
||||
@@ -1490,6 +1490,7 @@ export interface TranscodeVideoChunkParams {
|
||||
filePath?: string
|
||||
startTime: number
|
||||
endTime: number
|
||||
format?: 'video' | 'frames'
|
||||
}
|
||||
|
||||
export async function transcodeVideoChunkActivity(
|
||||
@@ -1502,6 +1503,82 @@ export async function transcodeVideoChunkActivity(
|
||||
})
|
||||
}
|
||||
|
||||
if (params.format === 'frames') {
|
||||
const tmpDir = transcodeService.createTempDir('video-chunk-frames-')
|
||||
try {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const inputSource =
|
||||
params.filePath ||
|
||||
(params.assetKey ? await s3Service.resolveInput(bucket, params.assetKey) : '')
|
||||
if (!inputSource) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
|
||||
|
||||
const duration = Math.max(0.1, params.endTime - params.startTime)
|
||||
const fpsFilter = duration > 16 ? `fps=${16 / duration}` : 'fps=1'
|
||||
const vframesArgs = duration > 16 ? ['-vframes', '16'] : []
|
||||
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'warning',
|
||||
...(isRemote
|
||||
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
|
||||
: []),
|
||||
'-ss',
|
||||
params.startTime.toString(),
|
||||
'-i',
|
||||
inputSource,
|
||||
'-map',
|
||||
'0:V:0',
|
||||
'-t',
|
||||
duration.toString(),
|
||||
'-vf',
|
||||
fpsFilter,
|
||||
...vframesArgs,
|
||||
'-q:v',
|
||||
'2',
|
||||
path.join(tmpDir, 'frame_%04d.jpg'),
|
||||
])
|
||||
|
||||
const frameFiles = fs
|
||||
.readdirSync(tmpDir)
|
||||
.filter((f) => /\.(jpe?g|png|webp)$/i.test(f))
|
||||
.sort()
|
||||
|
||||
const archiveEntries: Record<string, Buffer> = {}
|
||||
for (const file of frameFiles) {
|
||||
archiveEntries[file] = fs.readFileSync(path.join(tmpDir, file))
|
||||
}
|
||||
|
||||
const archive = new Bun.Archive(archiveEntries)
|
||||
const archiveBytes = await archive.bytes()
|
||||
const archiveData = Buffer.from(
|
||||
archiveBytes.buffer,
|
||||
archiveBytes.byteOffset,
|
||||
archiveBytes.byteLength,
|
||||
)
|
||||
const key = `files/${params.assetId}/tmp-embedding-chunks/frames-${params.startTime}-${params.endTime}-${ulid()}.tar`
|
||||
await s3Service.putObject(bucket, key, archiveData, archiveData.length, 'application/x-tar')
|
||||
|
||||
return { chunkKey: key }
|
||||
} catch (err) {
|
||||
if (err instanceof ApplicationFailure) {
|
||||
throw err
|
||||
}
|
||||
throw ApplicationFailure.create({
|
||||
message: `Failed to extract video chunk frames for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
nonRetryable: false,
|
||||
})
|
||||
} finally {
|
||||
transcodeService.removeDir(tmpDir)
|
||||
}
|
||||
}
|
||||
|
||||
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
|
||||
try {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
@@ -92,4 +92,39 @@ describe('AgentFormDialog', () => {
|
||||
expect(screen.getByText('Move Assets')).toBeDefined()
|
||||
expect(screen.getByText('Delete Asset')).toBeDefined()
|
||||
})
|
||||
|
||||
it('renders read-only state with system managed hint and only Close button', () => {
|
||||
render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<AgentFormDialog
|
||||
isOpen={true}
|
||||
readOnly={true}
|
||||
onClose={vi.fn()}
|
||||
teamId="team-123"
|
||||
type="embedding"
|
||||
title="Edit Embedding Agent"
|
||||
initialValues={{
|
||||
id: 'agent-1',
|
||||
name: 'embedding-agent',
|
||||
type: 'embedding',
|
||||
enabled: true,
|
||||
permission: 'reviewer',
|
||||
thinkingLevel: 'off',
|
||||
skills: [],
|
||||
mcpServerIds: [],
|
||||
deniedTools: [],
|
||||
readOnly: true,
|
||||
}}
|
||||
/>
|
||||
</QueryClientProvider>,
|
||||
)
|
||||
|
||||
expect(
|
||||
screen.getByText(
|
||||
'This agent is managed by the server environment configuration and cannot be modified or deleted.',
|
||||
),
|
||||
).toBeDefined()
|
||||
expect(screen.getAllByRole('button', { name: 'Close' }).length).toBe(2)
|
||||
expect(screen.queryByRole('button', { name: 'Save Changes' })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -48,6 +48,7 @@ interface AgentFormDialogProps {
|
||||
type?: AgentType
|
||||
initialValues?: AgentInfo
|
||||
title: string
|
||||
readOnly?: boolean
|
||||
}
|
||||
|
||||
const agentFormSchema = z.object({
|
||||
@@ -137,6 +138,7 @@ export function AgentFormDialog({
|
||||
type,
|
||||
initialValues,
|
||||
title,
|
||||
readOnly = false,
|
||||
}: AgentFormDialogProps) {
|
||||
const queryClient = useQueryClient()
|
||||
const { canAdmin } = usePermissions()
|
||||
@@ -368,6 +370,11 @@ export function AgentFormDialog({
|
||||
>
|
||||
<ScrollArea className="flex-1 min-h-0">
|
||||
<div className="p-6 pt-2 space-y-6">
|
||||
{readOnly && (
|
||||
<div className="p-3 bg-muted text-muted-foreground text-sm rounded-lg border border-border">
|
||||
{m.agent_managed_by_system_hint()}
|
||||
</div>
|
||||
)}
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
<form.Field
|
||||
name="name"
|
||||
@@ -382,6 +389,7 @@ export function AgentFormDialog({
|
||||
onChange={(e) => field.handleChange(e.target.value)}
|
||||
placeholder={m.agent_name_placeholder()}
|
||||
aria-invalid={isInvalid}
|
||||
disabled={readOnly}
|
||||
/>
|
||||
{isInvalid && <FieldError errors={mapErrors(field.state.meta.errors)} />}
|
||||
</Field>
|
||||
@@ -423,7 +431,7 @@ export function AgentFormDialog({
|
||||
onValueChange={(val) =>
|
||||
field.handleChange(val as 'owner' | 'editor' | 'reviewer')
|
||||
}
|
||||
disabled={!canAdmin}
|
||||
disabled={readOnly || !canAdmin}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
@@ -453,12 +461,15 @@ export function AgentFormDialog({
|
||||
<button
|
||||
key={preset.id}
|
||||
type="button"
|
||||
onClick={() => field.handleChange(preset.id)}
|
||||
disabled={readOnly}
|
||||
onClick={() => !readOnly && field.handleChange(preset.id)}
|
||||
className={cn(
|
||||
'relative w-14 h-14 rounded-full overflow-hidden border-2 transition-all hover:scale-105',
|
||||
'relative w-14 h-14 rounded-full overflow-hidden border-2 transition-all',
|
||||
!readOnly && 'hover:scale-105',
|
||||
isSelected
|
||||
? 'border-primary ring-2 ring-primary ring-offset-2'
|
||||
: 'border-border hover:border-muted-foreground',
|
||||
readOnly && 'cursor-default opacity-80',
|
||||
)}
|
||||
>
|
||||
<img
|
||||
@@ -473,7 +484,8 @@ export function AgentFormDialog({
|
||||
!AVAILABLE_AVATARS.some((p) => p.id === field.state.value) && (
|
||||
<button
|
||||
type="button"
|
||||
className="relative w-14 h-14 rounded-full overflow-hidden border-2 border-primary ring-2 ring-primary ring-offset-2 scale-105"
|
||||
disabled={true}
|
||||
className="relative w-14 h-14 rounded-full overflow-hidden border-2 border-primary ring-2 ring-primary ring-offset-2 scale-105 cursor-default"
|
||||
>
|
||||
<img
|
||||
src={field.state.value}
|
||||
@@ -1024,29 +1036,37 @@ export function AgentFormDialog({
|
||||
</ScrollArea>
|
||||
|
||||
<DialogFooter className="p-6 pt-4 border-t border-border flex-shrink-0">
|
||||
<Button variant="outline" onClick={onClose} type="button">
|
||||
{m.cancel()}
|
||||
</Button>
|
||||
<form.Subscribe
|
||||
selector={(state) => [state.canSubmit, state.isSubmitting]}
|
||||
children={([canSubmit, isSubmitting]) => (
|
||||
<Button
|
||||
type="submit"
|
||||
className="bg-primary hover:bg-primary/90 text-primary-foreground min-w-[120px]"
|
||||
disabled={
|
||||
!canSubmit ||
|
||||
isSubmitting ||
|
||||
createMutation.isPending ||
|
||||
updateMutation.isPending
|
||||
}
|
||||
>
|
||||
{(isSubmitting || createMutation.isPending || updateMutation.isPending) && (
|
||||
<Loader2 className="w-4 h-4 animate-spin mr-2" />
|
||||
)}
|
||||
{initialValues ? m.save_changes() : m.create_agent()}
|
||||
{readOnly ? (
|
||||
<Button variant="outline" onClick={onClose} type="button">
|
||||
{m.close()}
|
||||
</Button>
|
||||
) : (
|
||||
<>
|
||||
<Button variant="outline" onClick={onClose} type="button">
|
||||
{m.cancel()}
|
||||
</Button>
|
||||
)}
|
||||
/>
|
||||
<form.Subscribe
|
||||
selector={(state) => [state.canSubmit, state.isSubmitting]}
|
||||
children={([canSubmit, isSubmitting]) => (
|
||||
<Button
|
||||
type="submit"
|
||||
className="bg-primary hover:bg-primary/90 text-primary-foreground min-w-[120px]"
|
||||
disabled={
|
||||
!canSubmit ||
|
||||
isSubmitting ||
|
||||
createMutation.isPending ||
|
||||
updateMutation.isPending
|
||||
}
|
||||
>
|
||||
{(isSubmitting || createMutation.isPending || updateMutation.isPending) && (
|
||||
<Loader2 className="w-4 h-4 animate-spin mr-2" />
|
||||
)}
|
||||
{initialValues ? m.save_changes() : m.create_agent()}
|
||||
</Button>
|
||||
)}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</form>
|
||||
</DialogContent>
|
||||
|
||||
@@ -189,6 +189,7 @@ export function AgentsSettings({ teamId }: AgentsSettingsProps) {
|
||||
{(isCreateDialogOpen || editingAgent) && (
|
||||
<AgentFormDialog
|
||||
isOpen={true}
|
||||
readOnly={editingAgent?.readOnly}
|
||||
title={
|
||||
editingAgent
|
||||
? m.edit_item({ name: editingAgent.name })
|
||||
@@ -340,29 +341,38 @@ export function AgentsSettings({ teamId }: AgentsSettingsProps) {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<DropdownMenu modal={false}>
|
||||
<DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<MoreVertical className="w-4 h-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-32">
|
||||
<DropdownMenuItem
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
setAgentToDelete(agent)
|
||||
setIsDeleteDialogOpen(true)
|
||||
}}
|
||||
className="text-red-500 focus:text-red-500 gap-2 cursor-pointer"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" /> {m.delete()}
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
{agent.readOnly ? (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="text-xs text-muted-foreground border-border"
|
||||
>
|
||||
{m.agent_readonly()}
|
||||
</Badge>
|
||||
) : (
|
||||
<DropdownMenu modal={false}>
|
||||
<DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<MoreVertical className="w-4 h-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-32">
|
||||
<DropdownMenuItem
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
setAgentToDelete(agent)
|
||||
setIsDeleteDialogOpen(true)
|
||||
}}
|
||||
className="text-red-500 focus:text-red-500 gap-2 cursor-pointer"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" /> {m.delete()}
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between gap-2 pt-2 border-t border-border/50">
|
||||
@@ -408,6 +418,7 @@ export function AgentsSettings({ teamId }: AgentsSettingsProps) {
|
||||
<div onClick={(e) => e.stopPropagation()}>
|
||||
<Switch
|
||||
checked={agent.enabled}
|
||||
disabled={agent.readOnly}
|
||||
onCheckedChange={(enabled) => {
|
||||
updateAgentMutation.mutate({
|
||||
teamId,
|
||||
|
||||
@@ -633,6 +633,8 @@
|
||||
"chat_agent": "Chat Agent",
|
||||
"chat_agents": "Chat Agents",
|
||||
"embedding_agent": "Embedding Agent",
|
||||
"agent_readonly": "Read-only",
|
||||
"agent_managed_by_system_hint": "This agent is managed by the server environment configuration and cannot be modified or deleted.",
|
||||
"skills_count_value": "{count} Skills",
|
||||
"autofill_agent": "Autofill Agent",
|
||||
"ai_provider": "AI Provider",
|
||||
|
||||
@@ -633,6 +633,8 @@
|
||||
"chat_agent": "对话智能体",
|
||||
"chat_agents": "对话智能体",
|
||||
"embedding_agent": "嵌入智能体",
|
||||
"agent_readonly": "只读",
|
||||
"agent_managed_by_system_hint": "此智能体由服务器环境配置管理,无法进行修改或删除。",
|
||||
"skills_count_value": "{count} 个技能",
|
||||
"autofill_agent": "自动填充智能体",
|
||||
"ai_provider": "AI 提供商",
|
||||
|
||||
Reference in New Issue
Block a user