feat(agent): support gemma-embedding-2-openai provider (#475)

* feat(agent): support gemma-embedding-2-openai provider

Add support for local LiteRT-LM Gemma Embedding 2 provider with 768-dimensional vector embeddings, environment-configured read-only embedding agents, 16-frame uniform sampling for video chunks, and automatic WebP-to-JPEG conversion.

* refactor(agent): remove unused legacy generateEmbeddingActivity

* chore(agent): align sharp version to 0.35.4

* refactor(agent): rename embedding server env to GEMMA_EMBEDDING_OPENAI_BASE_URL

* refactor(agent,transcode): use native Bun.Archive for video chunk frames

* fix(transcode,agent): populate buffer entries in Bun.Archive and validate frame sizes
This commit is contained in:
Yiling-J
2026-10-15 15:49:24 +08:00
committed by GitHub
parent c302c162a0
commit b7271777e0
28 changed files with 1270 additions and 238 deletions
+2
View File
@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Added
* **agent**: Add support for local LiteRT-LM Gemma Embedding 2 (`gemma-embedding-2-openai`) provider and environment-enforced read-only embedding agents with automatic 768-dimensional vector search indexing
### Fixed
### Changed
+1
View File
@@ -19,6 +19,7 @@ async function start() {
process.env.BETTER_AUTH_URL = 'http://localhost:5200'
process.env.NODE_ENV = 'development'
process.env.SHUMAI_E2E = 'true'
process.env.ENABLE_EMBEDDING_AGENT = 'false'
// Write variables to .env.e2e file so the teardown can load them
const envContent = [`DATABASE_URL=${process.env.DATABASE_URL}`, `S3_BUCKET=${bucketName}`].join(
+2
View File
@@ -12,6 +12,7 @@ import { metadataService } from '@shumai/core/src/metadata/metadata'
import { initTranscodeWorkflows } from '@shumai/transcode'
import { workflowService } from '@shumai/workflow-core'
import { migrateLegacyAgentAvatars } from '@shumai/core/src/agent/migration'
import { agentService } from '@shumai/core/src/agent/agent'
import { handleDaemonCommands } from '@shumai/core/src/utils/daemon'
import { authService } from '@shumai/core/src/auth/auth'
@@ -58,6 +59,7 @@ async function run() {
// Start services
await metadataService.syncSystemFields().catch(console.error)
await migrateLegacyAgentAvatars().catch(console.error)
await agentService.ensureSystemEmbeddingAgents().catch(console.error)
assetService.startCleanupJob()
notificationJobService.start()
workflowService.start()
+1
View File
@@ -161,6 +161,7 @@
"@temporalio/workflow": "1.23.0",
"adm-zip": "0.6.0",
"jittered-fractional-indexing": "1.0.1",
"sharp": "0.35.4",
"typebox": "1.3.22",
"ulid": "3.0.2",
"zod": "4.5.4",
+1
View File
@@ -19,6 +19,7 @@
"@temporalio/workflow": "1.23.0",
"adm-zip": "0.6.0",
"jittered-fractional-indexing": "1.0.1",
"sharp": "0.35.4",
"typebox": "1.3.22",
"ulid": "3.0.2",
"zod": "4.5.4"
+315 -30
View File
@@ -1,4 +1,4 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
import {
generateTextEmbeddingActivity,
extractAiMetadataActivity,
@@ -6,12 +6,12 @@ import {
saveAssetEmbeddingsActivity,
generateImageEmbeddingActivity,
generateVideoChunkEmbeddingActivity,
generateEmbeddingActivity,
} from './ai'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
import { prisma, AssetType, AssetStatus } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import sharp from 'sharp'
vi.mock('fs', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- importOriginal returns the raw fs module, typed dynamically as any here
@@ -54,7 +54,6 @@ vi.mock('child_process', () => {
const execFileMock: any = vi.fn((file: any, args: any, cb: any) => {
cb(null, '120.0\n', '')
})
// Setup custom promisify behavior to return { stdout, stderr }
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mock functions dynamically return varying types based on test command calls
execFileMock[Symbol.for('nodejs.util.promisify.custom')] = vi.fn(async (file: any) => {
if (file === 'ffprobe') {
@@ -90,6 +89,14 @@ describe('AI Activities Unit Tests', () => {
beforeEach(() => {
vi.clearAllMocks()
process.env.GEMINI_API_KEY = 'test-key'
delete process.env.EMBEDDING_PROVIDER
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
})
afterEach(() => {
vi.unstubAllGlobals()
delete process.env.EMBEDDING_PROVIDER
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
})
it('should generate image embedding', async () => {
@@ -126,33 +133,6 @@ describe('AI Activities Unit Tests', () => {
)
})
it('should resolve video input via resolveInput and avoid downloadToFile in generateEmbeddingActivity', async () => {
vi.mocked(s3Service.downloadToFile).mockClear()
vi.mocked(s3Service.resolveInput).mockClear()
vi.mocked(s3Service.resolveInput).mockResolvedValue(
'https://mock-storage/files/asset-123/video.mp4',
)
const res = await generateEmbeddingActivity({
teamId: 't1',
assetId: 'asset-123',
context: {
agent: {},
asset: {
mediaType: 'video/mp4',
name: 'sample.mp4',
storageKey: { key: 'files/asset-123/video.mp4' },
media: { proxyType: 'video' },
},
},
})
expect(s3Service.resolveInput).toHaveBeenCalledWith('shumai', 'files/asset-123/video.mp4')
expect(s3Service.downloadToFile).not.toHaveBeenCalled()
expect(s3Service.getObject).not.toHaveBeenCalledWith('shumai', 'files/asset-123/video.mp4')
expect(res.embeddings.length).toBeGreaterThan(0)
})
it('should generate text embedding successfully', async () => {
const res = await generateTextEmbeddingActivity({
text: 'Search query',
@@ -172,6 +152,279 @@ describe('AI Activities Unit Tests', () => {
)
})
it('should generate text embedding successfully with gemma-embedding-2-openai', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.1) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 15 },
}),
})
vi.stubGlobal('fetch', mockFetch)
const res = await generateTextEmbeddingActivity({
text: 'Search query',
teamId: 't1',
})
expect(res.embedding).toHaveLength(768)
expect(res.usage.model).toBe('embeddinggemma-2')
expect(res.usage.inputTokens).toBe(15)
expect(mockFetch).toHaveBeenCalledWith(
'http://localhost:9379/v1/embeddings',
expect.objectContaining({
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: 'embeddinggemma-2',
input: 'task: search query | text: Search query',
}),
}),
)
})
it('should generate image embedding successfully with gemma-embedding-2-openai', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.2) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 140 },
}),
})
vi.stubGlobal('fetch', mockFetch)
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from('test-image'),
contentType: 'image/png',
} as unknown as { buffer: Buffer; contentType: string })
const res = await generateImageEmbeddingActivity({
teamId: 't1',
assetKey: 'img.png',
mediaType: 'image/png',
})
expect(res.embedding).toHaveLength(768)
expect(res.usage.model).toBe('embeddinggemma-2')
expect(res.usage.inputTokens).toBe(140)
expect(mockFetch).toHaveBeenCalledWith(
'http://localhost:9379/v1/embeddings',
expect.objectContaining({
method: 'POST',
body: JSON.stringify({
model: 'embeddinggemma-2',
input: [
{
type: 'image_url',
// eslint-disable-next-line @typescript-eslint/naming-convention
image_url: {
url: `data:image/png;base64,${Buffer.from('test-image').toString('base64')}`,
},
},
],
}),
}),
)
})
it('should respect custom GEMMA_EMBEDDING_OPENAI_BASE_URL with or without /v1 suffix', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.2) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 10 },
}),
})
vi.stubGlobal('fetch', mockFetch)
// Test with /v1
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379/v1'
await generateTextEmbeddingActivity({ text: 'test query', teamId: 't1' })
expect(mockFetch).toHaveBeenLastCalledWith(
'http://custom-host:9379/v1/embeddings',
expect.anything(),
)
// Test without /v1
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379'
await generateTextEmbeddingActivity({ text: 'test query', teamId: 't1' })
expect(mockFetch).toHaveBeenLastCalledWith(
'http://custom-host:9379/v1/embeddings',
expect.anything(),
)
})
it('should convert webp image to jpeg with gemma-embedding-2-openai', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.2) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 140 },
}),
})
vi.stubGlobal('fetch', mockFetch)
const webpBuffer = await sharp({
create: { width: 10, height: 10, channels: 3, background: { r: 255, g: 0, b: 0 } },
})
.webp()
.toBuffer()
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: webpBuffer,
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const res = await generateImageEmbeddingActivity({
teamId: 't1',
assetKey: 'img.webp',
mediaType: 'image/webp',
})
expect(res.embedding).toHaveLength(768)
expect(mockFetch).toHaveBeenCalledWith(
'http://localhost:9379/v1/embeddings',
expect.objectContaining({
method: 'POST',
body: expect.stringContaining('data:image/jpeg;base64,'),
}),
)
})
it('should generate video chunk embedding from tar archive with gemma-embedding-2-openai', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.3) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 140 },
}),
})
vi.stubGlobal('fetch', mockFetch)
const sampleArchive = new Bun.Archive({
'frame_0001.jpg': Buffer.from('frame-1'),
})
const archiveBytes = await sampleArchive.bytes()
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
contentType: 'application/x-tar',
} as unknown as { buffer: Buffer; contentType: string })
const res = await generateVideoChunkEmbeddingActivity({
teamId: 't1',
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
})
expect(res.embedding).toHaveLength(768)
expect(res.usage.model).toBe('embeddinggemma-2')
expect(mockFetch).toHaveBeenCalled()
})
it('should sample uniform 16 frames when chunk archive contains more than 16 frames', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: Array(768).fill(0.3) }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 140 },
}),
})
vi.stubGlobal('fetch', mockFetch)
const frames: Record<string, Buffer> = {}
for (let i = 1; i <= 30; i++) {
frames[`frame_${String(i).padStart(4, '0')}.jpg`] = Buffer.from(`frame-${i}`)
}
const sampleArchive = new Bun.Archive(frames)
const archiveBytes = await sampleArchive.bytes()
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
contentType: 'application/x-tar',
} as unknown as { buffer: Buffer; contentType: string })
const res = await generateVideoChunkEmbeddingActivity({
teamId: 't1',
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
})
expect(res.embedding).toHaveLength(768)
const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
expect(requestBody.input).toHaveLength(16)
})
it('should throw non-retryable ApplicationFailure when chunk archive contains no valid frames', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const sampleArchive = new Bun.Archive({
'info.txt': Buffer.from('not an image'),
})
const archiveBytes = await sampleArchive.bytes()
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
contentType: 'application/x-tar',
} as unknown as { buffer: Buffer; contentType: string })
await expect(
generateVideoChunkEmbeddingActivity({
teamId: 't1',
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
}),
).rejects.toThrow('No extracted frames found in chunk archive')
})
it('should throw non-retryable ApplicationFailure when chunk archive data is corrupted', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from('corrupted non-tar bytes'),
contentType: 'application/x-tar',
} as unknown as { buffer: Buffer; contentType: string })
await expect(
generateVideoChunkEmbeddingActivity({
teamId: 't1',
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
}),
).rejects.toThrow('Failed to read chunk archive')
})
it('should throw non-retryable ApplicationFailure when an extracted frame is empty (0 bytes)', async () => {
process.env.EMBEDDING_PROVIDER = 'gemma-embedding-2-openai'
const sampleArchive = new Bun.Archive({
'frame_0001.jpg': Buffer.alloc(0),
})
const archiveBytes = await sampleArchive.bytes()
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from(archiveBytes.buffer, archiveBytes.byteOffset, archiveBytes.byteLength),
contentType: 'application/x-tar',
} as unknown as { buffer: Buffer; contentType: string })
await expect(
generateVideoChunkEmbeddingActivity({
teamId: 't1',
chunkKey: 'files/asset-1/tmp-embedding-chunks/frames-0-60.tar',
}),
).rejects.toThrow('is empty (0 bytes)')
})
describe('extractAiMetadataActivity', () => {
it('should return existing images if they are already in S3 storage', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mock returned object cast as any due to partial implementation
@@ -423,5 +676,37 @@ describe('AI Database Activities Integration', () => {
expect(dbEmbeddings[0].endTime).toBe(60.0)
expect(dbEmbeddings[1].startTime).toBeNull()
})
it('should save 768-dim embeddings to embedding_768 column in database', async () => {
const embeddings = [
{
embedding: Array(768).fill(0.01),
startTime: 0.0,
endTime: 60.0,
},
{
embedding: Array(768).fill(0.02),
},
]
await saveAssetEmbeddingsActivity({
assetId: asset.id,
embeddings,
})
// Query raw embeddings from DB
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- db query output is a raw postgres result array typed as any
const dbEmbeddings: any[] = await prisma.$queryRaw`
SELECT id, asset_id as "assetId", (embedding_768 IS NOT NULL) as "has768", (embedding IS NOT NULL) as "has1536"
FROM asset_embeddings
WHERE asset_id = ${asset.id}
`
expect(dbEmbeddings.length).toBe(2)
expect(dbEmbeddings[0].has768).toBe(true)
expect(dbEmbeddings[0].has1536).toBe(false)
expect(dbEmbeddings[1].has768).toBe(true)
expect(dbEmbeddings[1].has1536).toBe(false)
})
})
})
+158 -140
View File
@@ -1,17 +1,11 @@
import { GoogleGenAI } from '@google/genai'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
import { execFile } from 'child_process'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import { promisify } from 'util'
import { ApplicationFailure } from '@temporalio/activity'
import { prisma } from '@shumai/db'
import { getProxyType } from '@shumai/core/src/utils/mime'
import { ulid } from 'ulid'
const execFileAsync = promisify(execFile)
import sharp from 'sharp'
export interface Usage {
inputTokens: number
@@ -25,15 +19,6 @@ export interface GeneratedEmbedding {
endTime?: number
}
export interface GenerateEmbeddingParams {
teamId: string
assetId: string
context: {
agent: unknown
asset: unknown
}
}
async function generateMultimodalEmbedding(
ai: GoogleGenAI,
buffer: Buffer,
@@ -62,131 +47,61 @@ async function generateMultimodalEmbedding(
return values
}
export async function generateEmbeddingActivity(params: GenerateEmbeddingParams): Promise<{
embeddings: GeneratedEmbedding[]
interface OpenAiEmbeddingItem {
type?: string
// eslint-disable-next-line @typescript-eslint/naming-convention
image_url?: {
url: string
}
}
async function callGemmaEmbeddingServer(input: string | OpenAiEmbeddingItem[]): Promise<{
embedding: number[]
usage: Usage
}> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- context properties are fetched as JSON from the DB activity
const { asset } = params.context as any
const rawBaseUrl = process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL || 'http://localhost:9379/v1'
if (!asset.mediaType) {
throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true })
}
const normalized = rawBaseUrl.replace(/\/+$/, '')
const endpoint = normalized.endsWith('/embeddings')
? normalized
: normalized.endsWith('/v1')
? `${normalized}/embeddings`
: `${normalized}/v1/embeddings`
const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
const response = await fetch(endpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: 'embeddinggemma-2',
input,
}),
})
if (!response.ok) {
const errorText = await response.text()
throw ApplicationFailure.create({
message: 'GEMINI_API_KEY environment variable is not configured',
nonRetryable: true,
message: `Embedding server returned status ${response.status}: ${errorText}`,
nonRetryable: false,
})
}
const resolvedApiKey = process.env[apiKey] || apiKey
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
const usage: Usage = {
model: 'gemini-embedding-2',
inputTokens: 0,
outputTokens: 0,
}
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
if (!isImage && !isVideo) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- response json shape is dynamic from external endpoint
const json: any = await response.json()
const embedding = json.data?.[0]?.embedding
if (!embedding || !Array.isArray(embedding)) {
throw ApplicationFailure.create({
message: `unsupported media type for embeddings: ${asset.mediaType}`,
nonRetryable: true,
message: 'Invalid embedding returned from embedding server',
nonRetryable: false,
})
}
const key = asset.storageKey?.key
if (!key) {
throw ApplicationFailure.create({ message: 'asset has no key', nonRetryable: true })
}
const bucket = process.env.S3_BUCKET || 'shumai'
const results: GeneratedEmbedding[] = []
if (isImage) {
const { buffer: data } = await s3Service.getObject(bucket, key)
const embVec = await generateMultimodalEmbedding(ai, data, asset.mediaType)
results.push({ embedding: embVec })
} else if (isVideo) {
const inputSource = await s3Service.resolveInput(bucket, key)
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
const { stdout } = await execFileAsync('ffprobe', [
'-v',
'error',
'-show_entries',
'format=duration',
'-of',
'default=noprint_wrappers=1:nokey=1',
inputSource,
])
const duration = parseFloat(stdout.trim())
if (isNaN(duration)) {
throw ApplicationFailure.create({
message: 'failed to parse video duration',
nonRetryable: true,
})
}
const chunkSize = 60.0
for (let start = 0.0; start < duration; start += chunkSize) {
let end = start + chunkSize
if (end > duration) end = duration
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
try {
await execFileAsync('ffmpeg', [
'-y',
'-loglevel',
'warning',
...(isRemote
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
: []),
'-ss',
start.toString(),
'-i',
inputSource,
'-map',
'0:V:0',
'-map',
'0:a:0?',
'-t',
(end - start).toString(),
'-c',
'copy',
chunkTmp,
])
const chunkData = fs.readFileSync(chunkTmp)
try {
const embVec = await generateMultimodalEmbedding(ai, chunkData, 'video/mp4')
results.push({ embedding: embVec, startTime: start, endTime: end })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (e: any) {
console.error(
`failed to generate video embedding for chunk ${start}-${end}: ${e.message}`,
)
}
} finally {
if (fs.existsSync(chunkTmp)) {
fs.unlinkSync(chunkTmp)
}
}
}
}
return {
embeddings: results,
usage,
embedding,
usage: {
model: json.model || 'gemma-embedding-2-openai',
inputTokens: json.usage?.prompt_tokens ?? 0,
outputTokens: 0,
},
}
}
@@ -199,6 +114,11 @@ export async function generateTextEmbeddingActivity(params: GenerateTextEmbeddin
embedding: number[]
usage: Usage
}> {
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
if (provider === 'gemma-embedding-2-openai') {
return callGemmaEmbeddingServer(`task: search query | text: ${params.text}`)
}
const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
throw ApplicationFailure.create({
@@ -339,12 +259,14 @@ export async function getEmbeddingContextActivity(params: { teamId: string; asse
const parsedOverlap = parseFloat(process.env.EMBEDDING_CHUNK_OVERLAP || '')
const chunkOverlap = Number.isFinite(parsedOverlap) && parsedOverlap >= 0 ? parsedOverlap : 5.0
const embeddingProvider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
return {
agent,
asset,
chunkDuration,
chunkOverlap,
embeddingProvider,
}
}
@@ -358,16 +280,32 @@ export async function saveAssetEmbeddingsActivity(params: {
}) {
for (const item of params.embeddings) {
const embVec = JSON.stringify(item.embedding)
const is768 = item.embedding.length === 768
if (item.startTime !== undefined && item.endTime !== undefined) {
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at)
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
`
if (is768) {
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, updated_at)
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
`
} else {
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at)
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW())
`
}
} else {
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at)
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
`
if (is768) {
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding_768, updated_at)
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
`
} else {
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at)
VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW())
`
}
}
}
}
@@ -384,6 +322,30 @@ export async function generateImageEmbeddingActivity(
embedding: number[]
usage: Usage
}> {
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
const { buffer: data } = await s3Service.getObject(
process.env.S3_BUCKET || 'shumai',
params.assetKey,
)
if (provider === 'gemma-embedding-2-openai') {
let imgData = data
let mimeType = params.mediaType || 'image/jpeg'
if (mimeType !== 'image/jpeg' && mimeType !== 'image/png') {
imgData = await sharp(data).jpeg().toBuffer()
mimeType = 'image/jpeg'
}
return callGemmaEmbeddingServer([
{
type: 'image_url',
// eslint-disable-next-line @typescript-eslint/naming-convention
image_url: {
url: `data:${mimeType};base64,${imgData.toString('base64')}`,
},
},
])
}
const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
throw ApplicationFailure.create({
@@ -401,10 +363,6 @@ export async function generateImageEmbeddingActivity(
outputTokens: 0,
}
const { buffer: data } = await s3Service.getObject(
process.env.S3_BUCKET || 'shumai',
params.assetKey,
)
const embVec = await generateMultimodalEmbedding(ai, data, params.mediaType)
return {
@@ -424,6 +382,66 @@ export async function generateVideoChunkEmbeddingActivity(
embedding: number[]
usage: Usage
}> {
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
const isGemma = provider === 'gemma-embedding-2-openai' || params.chunkKey.endsWith('.tar')
if (isGemma) {
const { buffer: data } = await s3Service.getObject(
process.env.S3_BUCKET || 'shumai',
params.chunkKey,
)
let filesMap: Map<string, File>
try {
const archive = new Bun.Archive(data)
filesMap = await archive.files()
} catch (err) {
throw ApplicationFailure.create({
message: `Failed to read chunk archive ${params.chunkKey}: ${err instanceof Error ? err.message : String(err)}`,
nonRetryable: true,
})
}
const frameFiles = Array.from(filesMap.keys())
.filter((f) => /\.(jpe?g|png|webp)$/i.test(f))
.sort()
if (frameFiles.length === 0) {
throw ApplicationFailure.create({
message: `No extracted frames found in chunk archive ${params.chunkKey}`,
nonRetryable: true,
})
}
let selectedFrames = frameFiles
if (selectedFrames.length > 16) {
const step = (selectedFrames.length - 1) / 15
selectedFrames = Array.from({ length: 16 }, (_, i) => selectedFrames[Math.round(i * step)])
}
const input: OpenAiEmbeddingItem[] = await Promise.all(
selectedFrames.map(async (f) => {
const file = filesMap.get(f)!
const fileBytes = await file.bytes()
if (fileBytes.byteLength === 0) {
throw ApplicationFailure.create({
message: `Extracted frame ${f} in chunk archive ${params.chunkKey} is empty (0 bytes)`,
nonRetryable: true,
})
}
return {
type: 'image_url',
// eslint-disable-next-line @typescript-eslint/naming-convention
image_url: {
url: `data:image/jpeg;base64,${Buffer.from(fileBytes).toString('base64')}`,
},
}
}),
)
return await callGemmaEmbeddingServer(input)
}
const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
throw ApplicationFailure.create({
-2
View File
@@ -625,13 +625,11 @@ export function initAgentWorkflows() {
export * from './activities/agent'
export {
extractAiMetadataActivity,
generateEmbeddingActivity,
generateImageEmbeddingActivity,
generateTextEmbeddingActivity,
generateVideoChunkEmbeddingActivity,
type ExtractAiMetadataParams,
type GeneratedEmbedding,
type GenerateEmbeddingParams,
type GenerateTextEmbeddingParams,
} from './activities/ai'
export * from './database-session-storage'
@@ -314,18 +314,21 @@ describe('Agent Embedding Workflow', () => {
assetKey: 'test-transcoded.mp4',
startTime: 0,
endTime: 60,
format: 'video',
})
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(2, {
assetId: 'a1',
assetKey: 'test-transcoded.mp4',
startTime: 55,
endTime: 115,
format: 'video',
})
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenNthCalledWith(3, {
assetId: 'a1',
assetKey: 'test-transcoded.mp4',
startTime: 110,
endTime: 150,
format: 'video',
})
// Verify video chunk embedding generation (3 chunks) on agent queue
@@ -485,4 +488,45 @@ describe('Agent Embedding Workflow', () => {
}),
)
})
it('should run agent embedding workflow for video with gemma-embedding-2-openai requesting frames format', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'a1',
mediaType: 'video/mp4',
storageKey: { key: 'test.mp4' },
media: {
duration: 30.0,
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
embeddingProvider: 'gemma-embedding-2-openai',
})
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'a1',
teamId: 't1',
},
})
await agentEmbeddingMedia(task)
expect(mockActivities.transcodeVideoChunkActivity).toHaveBeenCalledWith(
expect.objectContaining({
startTime: 0,
endTime: 30,
format: 'frames',
}),
)
expect(mockActivities.updateTaskUsageActivity).toHaveBeenCalledWith(
expect.objectContaining({
model: 'gemma-embedding-2-openai',
}),
)
})
})
@@ -68,7 +68,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
})
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const { asset, chunkDuration, chunkOverlap } = context as any
const { asset, chunkDuration, chunkOverlap, embeddingProvider } = context as any
const isGemmaProvider = embeddingProvider === 'gemma-embedding-2-openai'
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
const is3dWithViews =
@@ -115,7 +116,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
const results: GeneratedEmbedding[] = []
const usage = {
model: 'gemini-embedding-2',
model: isGemmaProvider ? 'gemma-embedding-2-openai' : 'gemini-embedding-2',
inputTokens: 0,
outputTokens: 0,
}
@@ -151,6 +152,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
assetKey: embeddingKey,
startTime: start,
endTime: end,
format: isGemmaProvider ? 'frames' : 'video',
})
const { chunkKey } = chunkRes
+118 -1
View File
@@ -1,4 +1,4 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
import { HTTPException } from 'hono/http-exception'
import { agentService } from '@shumai/core/src/agent/agent'
import { authzService } from '@shumai/core/src/authz/authz'
@@ -210,6 +210,30 @@ describe('Agent API', () => {
itemId: 'agent1',
})
})
it('returns 403 when updating embedding agent and ENABLE_EMBEDDING_AGENT is true', async () => {
process.env.ENABLE_EMBEDDING_AGENT = 'true'
try {
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
id: 'agent1',
teamId: 'team1',
type: 'embedding',
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
const res = await app.request('/agents/agent1', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: 'Updated Embedding Agent',
type: 'embedding',
}),
})
expect(res.status).toBe(403)
} finally {
delete process.env.ENABLE_EMBEDDING_AGENT
}
})
})
describe('DELETE /agents/:agentId', () => {
@@ -227,6 +251,25 @@ describe('Agent API', () => {
itemId: 'agent1',
})
})
it('returns 403 when deleting embedding agent and ENABLE_EMBEDDING_AGENT is true', async () => {
process.env.ENABLE_EMBEDDING_AGENT = 'true'
try {
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
id: 'agent1',
teamId: 'team1',
type: 'embedding',
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
const res = await app.request('/agents/agent1', {
method: 'DELETE',
})
expect(res.status).toBe(403)
} finally {
delete process.env.ENABLE_EMBEDDING_AGENT
}
})
})
describe('PATCH /agents/:agentId/permission', () => {
@@ -271,6 +314,27 @@ describe('Agent API', () => {
})
})
it('returns 403 when modifying embedding agent permission and ENABLE_EMBEDDING_AGENT is true', async () => {
process.env.ENABLE_EMBEDDING_AGENT = 'true'
try {
vi.mocked(agentService.getAgent).mockResolvedValueOnce({
id: 'agent1',
teamId: 'team1',
type: 'embedding',
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
const res = await app.request('/agents/agent1/permission', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ permission: 'owner' }),
})
expect(res.status).toBe(403)
} finally {
delete process.env.ENABLE_EMBEDDING_AGENT
}
})
it('denies access when user is not admin', async () => {
vi.mocked(authzService.hasPermission).mockRejectedValue(
new HTTPException(403, { message: 'Forbidden' }),
@@ -410,4 +474,57 @@ describe('Agent API', () => {
expect(agentService.listProjectChatAgents).not.toHaveBeenCalled()
})
})
describe('read-only embedding agent when ENABLE_EMBEDDING_AGENT is true', () => {
beforeEach(() => {
process.env.ENABLE_EMBEDDING_AGENT = 'true'
vi.mocked(agentService.getAgent).mockResolvedValue({
id: 'emb1',
teamId: 'team1',
type: 'embedding',
} as unknown as Awaited<ReturnType<typeof agentService.getAgent>>)
})
afterEach(() => {
delete process.env.ENABLE_EMBEDDING_AGENT
})
it('rejects PUT /agents/:agentId with 403', async () => {
const res = await app.request('/agents/emb1', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: 'Updated Embedding',
type: 'embedding',
enabled: false,
}),
})
expect(res.status).toBe(403)
expect(agentService.updateAgent).not.toHaveBeenCalled()
})
it('rejects DELETE /agents/:agentId with 403', async () => {
const res = await app.request('/agents/emb1', {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' },
})
expect(res.status).toBe(403)
expect(agentService.deleteAgent).not.toHaveBeenCalled()
})
it('rejects PATCH /agents/:agentId/permission with 403', async () => {
const res = await app.request('/agents/emb1/permission', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
permission: 'owner',
}),
})
expect(res.status).toBe(403)
expect(agentService.updateAgentPermission).not.toHaveBeenCalled()
})
})
})
+20
View File
@@ -1,4 +1,5 @@
import { Hono } from 'hono'
import { HTTPException } from 'hono/http-exception'
import { zValidator } from '@hono/zod-validator'
import { agentService } from '@shumai/core/src/agent/agent'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
@@ -92,6 +93,13 @@ const route = new Hono<{ Variables: { user: User } }>()
id: agentId,
})
const existingAgent = await agentService.getAgent({ agentId })
if (existingAgent?.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true') {
throw new HTTPException(403, {
message: 'Embedding agent is managed by environment configuration and cannot be modified',
})
}
const agent = await agentService.updateAgent({
agentId,
...req,
@@ -123,6 +131,13 @@ const route = new Hono<{ Variables: { user: User } }>()
id: agentId,
})
const existingAgent = await agentService.getAgent({ agentId })
if (existingAgent?.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true') {
throw new HTTPException(403, {
message: 'Embedding agent is managed by environment configuration and cannot be modified',
})
}
const agent = await agentService.updateAgentPermission(agentId, req.permission)
await auditLogService.logAction({
@@ -149,6 +164,11 @@ const route = new Hono<{ Variables: { user: User } }>()
})
const existingAgent = await agentService.getAgent({ agentId })
if (existingAgent?.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true') {
throw new HTTPException(403, {
message: 'Embedding agent is managed by environment configuration and cannot be deleted',
})
}
await agentService.deleteAgent({
agentId,
+81
View File
@@ -976,4 +976,85 @@ describe('AgentService', () => {
expect(ids).toHaveLength(0)
})
})
describe('ensureEmbeddingAgent', () => {
test('creates embedding agent when ENABLE_EMBEDDING_AGENT is true', async () => {
const db = prisma
const svc = new AgentService()
const { team } = await setupTestData(db)
process.env.ENABLE_EMBEDDING_AGENT = 'true'
try {
await svc.ensureEmbeddingAgentForTeam(team.id)
const agent = await db.agent.findFirst({
where: { teamId: team.id, type: 'embedding' },
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
})
expect(agent).toBeDefined()
const info = await svc.toAgentInfo(agent!)
expect(info.name).toBe('embedding-agent')
expect(info.type).toBe('embedding')
expect(info.enabled).toBe(true)
expect(info.avatarPreset).toBe('avatar-1')
expect(info.avatar).toContain('avatar-1.webp')
expect(info.readOnly).toBe(true)
// Ensure idempotency
await svc.ensureEmbeddingAgentForTeam(team.id)
const count = await db.agent.count({
where: { teamId: team.id, type: 'embedding' },
})
expect(count).toBe(1)
} finally {
delete process.env.ENABLE_EMBEDDING_AGENT
}
})
test('enables existing embedding agent when ENABLE_EMBEDDING_AGENT is true', async () => {
const db = prisma
const svc = new AgentService()
const { team } = await setupTestData(db)
const bot = await db.user.create({
data: { name: 'Existing Embedding', email: 'existing-emb@shumai.ai', type: 'agent' },
})
await db.agent.create({
data: {
id: bot.id,
teamId: team.id,
type: 'embedding',
enabled: false,
config: { provider: '', model: '' },
},
})
process.env.ENABLE_EMBEDDING_AGENT = 'true'
try {
await svc.ensureEmbeddingAgentForTeam(team.id)
const agent = await db.agent.findFirst({
where: { teamId: team.id, type: 'embedding' },
include: { user: true, skills: { include: { skill: true } }, mcpServers: true },
})
expect(agent?.id).toBe(bot.id)
expect(agent?.enabled).toBe(true)
const info = await svc.toAgentInfo(agent!)
expect(info.readOnly).toBe(true)
} finally {
delete process.env.ENABLE_EMBEDDING_AGENT
}
})
test('does nothing when ENABLE_EMBEDDING_AGENT is false or not set', async () => {
const db = prisma
const svc = new AgentService()
const team = await db.team.create({ data: { name: 'Team No Agent' } })
delete process.env.ENABLE_EMBEDDING_AGENT
await svc.ensureEmbeddingAgentForTeam(team.id)
const agent = await db.agent.findFirst({
where: { teamId: team.id, type: 'embedding' },
})
expect(agent).toBeNull()
})
})
})
+49 -3
View File
@@ -43,8 +43,8 @@ interface AgentInfoSource {
config: unknown
soul: string | null
user: { name: string; image: string | null }
skills: Array<{ id: string; skillId: string; skill: { id: string; name: string } }>
mcpServers: Array<{ mcpServerId: string }>
skills?: Array<{ id: string; skillId: string; skill: { id: string; name: string } }>
mcpServers?: Array<{ mcpServerId: string }>
}
function extractS3Key(urlOrKey?: string | null): string | null | undefined {
@@ -462,6 +462,8 @@ export class AgentService {
*/
async toAgentInfo(agent: AgentInfoSource): Promise<AgentInfo> {
const config = agent.config as unknown as PrismaJson.AgentConfig
const isReadOnly = agent.type === 'embedding' && process.env.ENABLE_EMBEDDING_AGENT === 'true'
return {
id: agent.id,
name: agent.user.name,
@@ -475,13 +477,14 @@ export class AgentService {
thinkingLevel: config.thinkingLevel || 'off',
systemPrompt: config.systemPrompt,
soul: agent.soul || undefined,
skills: agent.skills.map((s) => ({
skills: (agent.skills || []).map((s) => ({
id: s.id,
skillId: s.skillId,
skill: s.skill,
})),
mcpServerIds: (agent.mcpServers || []).map((m) => m.mcpServerId),
deniedTools: config.deniedTools || [],
readOnly: isReadOnly,
}
}
@@ -718,6 +721,49 @@ export class AgentService {
params,
)
}
async ensureSystemEmbeddingAgents(): Promise<void> {
if (process.env.ENABLE_EMBEDDING_AGENT !== 'true') {
return
}
const teams = await this.prismaClient.team.findMany({
select: { id: true },
})
for (const team of teams) {
await this.ensureEmbeddingAgentForTeam(team.id)
}
}
async ensureEmbeddingAgentForTeam(teamId: string): Promise<void> {
if (process.env.ENABLE_EMBEDDING_AGENT !== 'true') {
return
}
const existing = await this.prismaClient.agent.findFirst({
where: {
teamId,
type: 'embedding',
},
})
if (!existing) {
await this.createAgent({
teamId,
name: 'embedding-agent',
type: 'embedding',
avatar: 'avatar-1',
thinkingLevel: 'off',
enabled: true,
})
} else if (!existing.enabled) {
await this.prismaClient.agent.update({
where: { id: existing.id },
data: { enabled: true },
})
}
}
}
export const agentService = new AgentService()
+148
View File
@@ -857,6 +857,154 @@ describe('SearchService', () => {
mockExecuteWait.mockRestore()
})
it('matches and sorts 768-dimensional embeddings using embedding_768 column', async () => {
const user = await prisma.user.create({
data: { name: 'semantic-user-768', email: 'semantic-user-768@example.com', type: 'human' },
})
const team = await prisma.team.create({
data: { name: 'semantic-team-768' },
})
const project = await prisma.project.create({
data: { name: 'semantic-project-768', teamId: team.id },
})
const botUser = await prisma.user.create({
data: { name: 'Embedding Bot 768', email: 'emb-768@example.com', type: 'agent' },
})
const provider = await prisma.provider.create({
data: {
name: 'gemma',
teamId: team.id,
config: { api: 'openai', apiKey: 'key' } as unknown as PrismaJson.ProviderConfig,
},
})
const model = await prisma.model.create({
data: {
modelId: 'embeddinggemma-2',
name: 'EmbeddingGemma 2',
providerId: provider.id,
config: {
reasoning: false,
input: ['text'],
contextWindow: 1000,
maxTokens: 1000,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
} as unknown as PrismaJson.ModelConfig,
},
})
await prisma.agent.create({
data: {
id: botUser.id,
teamId: team.id,
type: 'embedding',
enabled: true,
providerId: provider.id,
modelId: model.id,
config: { provider: 'gemma-embedding-2-openai', model: 'embeddinggemma-2' },
},
})
const rootFolder = await prisma.asset.create({
data: {
name: 'root-768',
type: AssetType.folder,
projectId: project.id,
creatorId: user.id,
status: 'uploaded',
},
})
const videoAssetA = await prisma.asset.create({
data: {
name: 'VideoA_768.mp4',
type: AssetType.file,
mediaType: 'video/mp4',
projectId: project.id,
parentId: rootFolder.id,
creatorId: user.id,
status: 'uploaded',
},
})
const videoAssetB = await prisma.asset.create({
data: {
name: 'VideoB_768.mp4',
type: AssetType.file,
mediaType: 'video/mp4',
projectId: project.id,
parentId: rootFolder.id,
creatorId: user.id,
status: 'uploaded',
},
})
const arrA1 = Array(768).fill(0)
arrA1[0] = 1.0
const vectorA1 = JSON.stringify(arrA1)
const arrB1 = Array(768).fill(0)
arrB1[0] = 1.0
arrB1[1] = 1.0
const vectorB1 = JSON.stringify(arrB1)
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, created_at, updated_at)
VALUES (
'chunk-768-a1',
${videoAssetA.id},
${vectorA1}::vector,
0.0,
10.0,
NOW(),
NOW()
)
`
await prisma.$executeRaw`
INSERT INTO asset_embeddings (id, asset_id, embedding_768, start_time, end_time, created_at, updated_at)
VALUES (
'chunk-768-b1',
${videoAssetB.id},
${vectorB1}::vector,
5.0,
15.0,
NOW(),
NOW()
)
`
// Query vector with 768 dims pointing along dimension 0
const queryVector = Array(768).fill(0)
queryVector[0] = 1.0
const mockExecuteWait = vi
.spyOn(workflowService, 'executeWait')
.mockImplementation(async (task) => {
return {
...task,
status: 'completed',
output: {
embedding: queryVector,
},
} as unknown as typeof task
})
const result = await searchService.search(rootFolder.id, {
recursively: true,
assetType: 'file',
operator: 'AND',
conditions: [],
query: 'find 768 dim match',
isSemantic: true,
})
expect(result.data.length).toBeGreaterThanOrEqual(2)
expect(result.data[0].id).toBe(videoAssetA.id)
expect(result.data[1].id).toBe(videoAssetB.id)
mockExecuteWait.mockRestore()
})
})
describe('totalSize calculation', () => {
+11 -1
View File
@@ -68,12 +68,21 @@ export class SearchService {
// 3. Construct raw SQL using SqlQueryBuilder
const vectorJson = JSON.stringify(queryVector)
const is768 = queryVector.length === 768
const distanceExpr = is768
? Prisma.sql`(ae.embedding_768 <=> ${vectorJson}::vector)`
: Prisma.sql`(ae.embedding <=> ${vectorJson}::vector)`
const notNullCond = is768
? Prisma.sql`ae.embedding_768 IS NOT NULL`
: Prisma.sql`ae.embedding IS NOT NULL`
const builder = new SqlQueryBuilder()
.select(
Prisma.sql`a.id as "assetId", ae.start_time as "startTime", ae.end_time as "endTime", (ae.embedding <=> ${vectorJson}::vector) as "distance"`,
Prisma.sql`a.id as "assetId", ae.start_time as "startTime", ae.end_time as "endTime", ${distanceExpr} as "distance"`,
)
.from(Prisma.sql`assets a JOIN asset_embeddings ae ON a.id = ae.asset_id`)
.addWhere(Prisma.sql`a.is_deleted = false`)
.addWhere(notNullCond)
if (targetFolderIds.length > 0) {
builder.addWhere(Prisma.sql`a.parent_id = ANY(${targetFolderIds})`)
@@ -158,6 +167,7 @@ export class SearchService {
.select(Prisma.sql`COUNT(*)`)
.from(Prisma.sql`assets a JOIN asset_embeddings ae ON a.id = ae.asset_id`)
.addWhere(Prisma.sql`a.is_deleted = false`)
.addWhere(notNullCond)
if (targetFolderIds.length > 0) {
countBuilder.addWhere(Prisma.sql`a.parent_id = ANY(${targetFolderIds})`)
+5 -1
View File
@@ -6,6 +6,7 @@ import { notificationService } from '@shumai/core/src/notification/notification'
import { getAvatarUrl } from '@shumai/core/src/user/avatar'
import { HTTPException } from 'hono/http-exception'
import { getAllowedAgentRoles } from '@shumai/core/src/agent/permissions'
import { AgentService, agentService } from '@shumai/core/src/agent/agent'
import { sandboxService } from '@shumai/core/src/sandbox/sandbox-service'
import { turntableService } from '@shumai/core/src/turntable/turntable'
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
@@ -46,6 +47,7 @@ export class TeamService {
},
})
await providerService.initBuiltinProviders(team.id)
await agentService.ensureEmbeddingAgentForTeam(team.id)
}
return team
@@ -86,10 +88,12 @@ export class TeamService {
},
})
// We use 'any' here because ProviderService expects the full PrismaClient,
// We use 'any' here because ProviderService/AgentService expect the full PrismaClient,
// but the TransactionClient is compatible with the methods we use.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await new ProviderService(tx as any).initBuiltinProviders(team.id)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await new AgentService(tx as any).ensureEmbeddingAgentForTeam(team.id)
return this.toTeamInfo(updatedTeam)
})
@@ -0,0 +1,8 @@
-- AlterTable
ALTER TABLE "asset_embeddings" ADD COLUMN "embedding_768" vector(768);
-- CreateIndex: HNSW index on asset_embeddings.embedding_768
CREATE INDEX "AssetEmbedding_embedding_768_idx"
ON "asset_embeddings"
USING hnsw (embedding_768 vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
+10 -8
View File
@@ -706,17 +706,19 @@ model AssetCommentReaction {
}
model AssetEmbedding {
id String @id @default(ulid())
embedding Unsupported("vector(1536)")?
startTime Float? @map("start_time")
endTime Float? @map("end_time")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
id String @id @default(ulid())
embedding Unsupported("vector(1536)")?
embedding768 Unsupported("vector(768)")? @map("embedding_768")
startTime Float? @map("start_time")
endTime Float? @map("end_time")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
assetId String @map("asset_id")
asset Asset @relation("AssetToEmbedding", fields: [assetId], references: [id], onDelete: Cascade)
assetId String @map("asset_id")
asset Asset @relation("AssetToEmbedding", fields: [assetId], references: [id], onDelete: Cascade)
@@index([embedding], name: "AssetEmbedding_embedding_idx")
@@index([embedding768], name: "AssetEmbedding_embedding_768_idx")
@@map("asset_embeddings")
}
File diff suppressed because one or more lines are too long
+1
View File
@@ -42,6 +42,7 @@ export const agentInfoSchema = z.object({
skills: z.array(agentSkillSchema).optional(),
mcpServerIds: z.array(z.string()).optional(),
deniedTools: z.array(z.string()).optional(),
readOnly: z.boolean().optional(),
})
export type AgentInfo = z.infer<typeof agentInfoSchema>
@@ -1230,6 +1230,100 @@ describe('Transcode Activities', () => {
expect(res.chunkKey).toMatch(/^files\/asset-abc\/tmp-embedding-chunks\/chunk-10-25-.*\.mp4$/)
})
it('should extract frames archive when format is frames', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(
_file: string,
args: string[],
cb: (err: Error | null, result: { stdout: string; stderr: string }) => void,
) => {
if (_file === 'ffmpeg' && args) {
const outPattern = args[args.length - 1]
if (outPattern && outPattern.includes('frame_%04d.jpg')) {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const fsActual = require('fs')
// eslint-disable-next-line @typescript-eslint/no-require-imports
const pathActual = require('path')
const dir = pathActual.dirname(outPattern)
fsActual.writeFileSync(
pathActual.join(dir, 'frame_0001.jpg'),
Buffer.from('mock-frame'),
)
}
}
cb(null, { stdout: '', stderr: '' })
},
)
const res = await transcodeVideoChunkActivity({
assetId: 'asset-abc',
filePath: '/tmp/full-video.mp4',
startTime: 10,
endTime: 25,
format: 'frames',
})
expect(child_process.execFile).toHaveBeenCalledWith(
'ffmpeg',
expect.arrayContaining(['-vf', 'fps=1', '-q:v', '2']),
expect.any(Function),
)
expect(child_process.execFile).not.toHaveBeenCalledWith(
'tar',
expect.anything(),
expect.anything(),
)
expect(s3Service.putObject).toHaveBeenCalledWith(
'shumai',
expect.stringMatching(/^files\/asset-abc\/tmp-embedding-chunks\/frames-10-25-.*\.tar$/),
expect.any(Buffer),
expect.any(Number),
'application/x-tar',
)
expect(res.chunkKey).toMatch(/^files\/asset-abc\/tmp-embedding-chunks\/frames-10-25-.*\.tar$/)
const putCall = vi.mocked(s3Service.putObject).mock.calls[0]
const uploadedBuffer = putCall[2] as Buffer
const uploadedArchive = new Bun.Archive(uploadedBuffer)
const archiveFiles = await uploadedArchive.files()
expect(archiveFiles.has('frame_0001.jpg')).toBe(true)
const uploadedFile = archiveFiles.get('frame_0001.jpg')!
expect(uploadedFile.size).toBeGreaterThan(0)
const uploadedFileBytes = await uploadedFile.bytes()
expect(uploadedFileBytes.byteLength).toBeGreaterThan(0)
})
it('should sample uniform 16 frames when chunk duration > 16s', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- child_process.execFile mockImplementation returns void rather than ChildProcess in unit tests
;(child_process.execFile as any).mockImplementation(
(
_cmd: unknown,
_args: unknown,
cb: (err: null, res: { stdout: string; stderr: string }) => void,
) => {
cb(null, {
stdout: '',
stderr: '',
})
},
)
await transcodeVideoChunkActivity({
assetId: 'asset-abc',
filePath: '/tmp/full-video.mp4',
startTime: 0,
endTime: 60,
format: 'frames',
})
expect(child_process.execFile).toHaveBeenCalledWith(
'ffmpeg',
expect.arrayContaining(['-vf', `fps=${16 / 60}`, '-vframes', '16', '-q:v', '2']),
expect.any(Function),
)
})
it('should throw non-retryable ApplicationFailure when neither filePath nor assetKey is provided', async () => {
expect.assertions(2)
try {
@@ -1490,6 +1490,7 @@ export interface TranscodeVideoChunkParams {
filePath?: string
startTime: number
endTime: number
format?: 'video' | 'frames'
}
export async function transcodeVideoChunkActivity(
@@ -1502,6 +1503,82 @@ export async function transcodeVideoChunkActivity(
})
}
if (params.format === 'frames') {
const tmpDir = transcodeService.createTempDir('video-chunk-frames-')
try {
const bucket = process.env.S3_BUCKET || 'shumai'
const inputSource =
params.filePath ||
(params.assetKey ? await s3Service.resolveInput(bucket, params.assetKey) : '')
if (!inputSource) {
throw ApplicationFailure.create({
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
nonRetryable: true,
})
}
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
const duration = Math.max(0.1, params.endTime - params.startTime)
const fpsFilter = duration > 16 ? `fps=${16 / duration}` : 'fps=1'
const vframesArgs = duration > 16 ? ['-vframes', '16'] : []
await execFileAsync('ffmpeg', [
'-y',
'-loglevel',
'warning',
...(isRemote
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
: []),
'-ss',
params.startTime.toString(),
'-i',
inputSource,
'-map',
'0:V:0',
'-t',
duration.toString(),
'-vf',
fpsFilter,
...vframesArgs,
'-q:v',
'2',
path.join(tmpDir, 'frame_%04d.jpg'),
])
const frameFiles = fs
.readdirSync(tmpDir)
.filter((f) => /\.(jpe?g|png|webp)$/i.test(f))
.sort()
const archiveEntries: Record<string, Buffer> = {}
for (const file of frameFiles) {
archiveEntries[file] = fs.readFileSync(path.join(tmpDir, file))
}
const archive = new Bun.Archive(archiveEntries)
const archiveBytes = await archive.bytes()
const archiveData = Buffer.from(
archiveBytes.buffer,
archiveBytes.byteOffset,
archiveBytes.byteLength,
)
const key = `files/${params.assetId}/tmp-embedding-chunks/frames-${params.startTime}-${params.endTime}-${ulid()}.tar`
await s3Service.putObject(bucket, key, archiveData, archiveData.length, 'application/x-tar')
return { chunkKey: key }
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
throw ApplicationFailure.create({
message: `Failed to extract video chunk frames for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
nonRetryable: false,
})
} finally {
transcodeService.removeDir(tmpDir)
}
}
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
try {
const bucket = process.env.S3_BUCKET || 'shumai'
@@ -92,4 +92,39 @@ describe('AgentFormDialog', () => {
expect(screen.getByText('Move Assets')).toBeDefined()
expect(screen.getByText('Delete Asset')).toBeDefined()
})
it('renders read-only state with system managed hint and only Close button', () => {
render(
<QueryClientProvider client={queryClient}>
<AgentFormDialog
isOpen={true}
readOnly={true}
onClose={vi.fn()}
teamId="team-123"
type="embedding"
title="Edit Embedding Agent"
initialValues={{
id: 'agent-1',
name: 'embedding-agent',
type: 'embedding',
enabled: true,
permission: 'reviewer',
thinkingLevel: 'off',
skills: [],
mcpServerIds: [],
deniedTools: [],
readOnly: true,
}}
/>
</QueryClientProvider>,
)
expect(
screen.getByText(
'This agent is managed by the server environment configuration and cannot be modified or deleted.',
),
).toBeDefined()
expect(screen.getAllByRole('button', { name: 'Close' }).length).toBe(2)
expect(screen.queryByRole('button', { name: 'Save Changes' })).toBeNull()
})
})
@@ -48,6 +48,7 @@ interface AgentFormDialogProps {
type?: AgentType
initialValues?: AgentInfo
title: string
readOnly?: boolean
}
const agentFormSchema = z.object({
@@ -137,6 +138,7 @@ export function AgentFormDialog({
type,
initialValues,
title,
readOnly = false,
}: AgentFormDialogProps) {
const queryClient = useQueryClient()
const { canAdmin } = usePermissions()
@@ -368,6 +370,11 @@ export function AgentFormDialog({
>
<ScrollArea className="flex-1 min-h-0">
<div className="p-6 pt-2 space-y-6">
{readOnly && (
<div className="p-3 bg-muted text-muted-foreground text-sm rounded-lg border border-border">
{m.agent_managed_by_system_hint()}
</div>
)}
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
<form.Field
name="name"
@@ -382,6 +389,7 @@ export function AgentFormDialog({
onChange={(e) => field.handleChange(e.target.value)}
placeholder={m.agent_name_placeholder()}
aria-invalid={isInvalid}
disabled={readOnly}
/>
{isInvalid && <FieldError errors={mapErrors(field.state.meta.errors)} />}
</Field>
@@ -423,7 +431,7 @@ export function AgentFormDialog({
onValueChange={(val) =>
field.handleChange(val as 'owner' | 'editor' | 'reviewer')
}
disabled={!canAdmin}
disabled={readOnly || !canAdmin}
>
<SelectTrigger>
<SelectValue />
@@ -453,12 +461,15 @@ export function AgentFormDialog({
<button
key={preset.id}
type="button"
onClick={() => field.handleChange(preset.id)}
disabled={readOnly}
onClick={() => !readOnly && field.handleChange(preset.id)}
className={cn(
'relative w-14 h-14 rounded-full overflow-hidden border-2 transition-all hover:scale-105',
'relative w-14 h-14 rounded-full overflow-hidden border-2 transition-all',
!readOnly && 'hover:scale-105',
isSelected
? 'border-primary ring-2 ring-primary ring-offset-2'
: 'border-border hover:border-muted-foreground',
readOnly && 'cursor-default opacity-80',
)}
>
<img
@@ -473,7 +484,8 @@ export function AgentFormDialog({
!AVAILABLE_AVATARS.some((p) => p.id === field.state.value) && (
<button
type="button"
className="relative w-14 h-14 rounded-full overflow-hidden border-2 border-primary ring-2 ring-primary ring-offset-2 scale-105"
disabled={true}
className="relative w-14 h-14 rounded-full overflow-hidden border-2 border-primary ring-2 ring-primary ring-offset-2 scale-105 cursor-default"
>
<img
src={field.state.value}
@@ -1024,29 +1036,37 @@ export function AgentFormDialog({
</ScrollArea>
<DialogFooter className="p-6 pt-4 border-t border-border flex-shrink-0">
<Button variant="outline" onClick={onClose} type="button">
{m.cancel()}
</Button>
<form.Subscribe
selector={(state) => [state.canSubmit, state.isSubmitting]}
children={([canSubmit, isSubmitting]) => (
<Button
type="submit"
className="bg-primary hover:bg-primary/90 text-primary-foreground min-w-[120px]"
disabled={
!canSubmit ||
isSubmitting ||
createMutation.isPending ||
updateMutation.isPending
}
>
{(isSubmitting || createMutation.isPending || updateMutation.isPending) && (
<Loader2 className="w-4 h-4 animate-spin mr-2" />
)}
{initialValues ? m.save_changes() : m.create_agent()}
{readOnly ? (
<Button variant="outline" onClick={onClose} type="button">
{m.close()}
</Button>
) : (
<>
<Button variant="outline" onClick={onClose} type="button">
{m.cancel()}
</Button>
)}
/>
<form.Subscribe
selector={(state) => [state.canSubmit, state.isSubmitting]}
children={([canSubmit, isSubmitting]) => (
<Button
type="submit"
className="bg-primary hover:bg-primary/90 text-primary-foreground min-w-[120px]"
disabled={
!canSubmit ||
isSubmitting ||
createMutation.isPending ||
updateMutation.isPending
}
>
{(isSubmitting || createMutation.isPending || updateMutation.isPending) && (
<Loader2 className="w-4 h-4 animate-spin mr-2" />
)}
{initialValues ? m.save_changes() : m.create_agent()}
</Button>
)}
/>
</>
)}
</DialogFooter>
</form>
</DialogContent>
@@ -189,6 +189,7 @@ export function AgentsSettings({ teamId }: AgentsSettingsProps) {
{(isCreateDialogOpen || editingAgent) && (
<AgentFormDialog
isOpen={true}
readOnly={editingAgent?.readOnly}
title={
editingAgent
? m.edit_item({ name: editingAgent.name })
@@ -340,29 +341,38 @@ export function AgentsSettings({ teamId }: AgentsSettingsProps) {
</div>
</div>
</div>
<DropdownMenu modal={false}>
<DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-foreground"
>
<MoreVertical className="w-4 h-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-32">
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation()
setAgentToDelete(agent)
setIsDeleteDialogOpen(true)
}}
className="text-red-500 focus:text-red-500 gap-2 cursor-pointer"
>
<Trash2 className="w-4 h-4" /> {m.delete()}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
{agent.readOnly ? (
<Badge
variant="outline"
className="text-xs text-muted-foreground border-border"
>
{m.agent_readonly()}
</Badge>
) : (
<DropdownMenu modal={false}>
<DropdownMenuTrigger asChild onClick={(e) => e.stopPropagation()}>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-foreground"
>
<MoreVertical className="w-4 h-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-32">
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation()
setAgentToDelete(agent)
setIsDeleteDialogOpen(true)
}}
className="text-red-500 focus:text-red-500 gap-2 cursor-pointer"
>
<Trash2 className="w-4 h-4" /> {m.delete()}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
)}
</div>
<div className="flex items-center justify-between gap-2 pt-2 border-t border-border/50">
@@ -408,6 +418,7 @@ export function AgentsSettings({ teamId }: AgentsSettingsProps) {
<div onClick={(e) => e.stopPropagation()}>
<Switch
checked={agent.enabled}
disabled={agent.readOnly}
onCheckedChange={(enabled) => {
updateAgentMutation.mutate({
teamId,
+2
View File
@@ -633,6 +633,8 @@
"chat_agent": "Chat Agent",
"chat_agents": "Chat Agents",
"embedding_agent": "Embedding Agent",
"agent_readonly": "Read-only",
"agent_managed_by_system_hint": "This agent is managed by the server environment configuration and cannot be modified or deleted.",
"skills_count_value": "{count} Skills",
"autofill_agent": "Autofill Agent",
"ai_provider": "AI Provider",
+2
View File
@@ -633,6 +633,8 @@
"chat_agent": "对话智能体",
"chat_agents": "对话智能体",
"embedding_agent": "嵌入智能体",
"agent_readonly": "只读",
"agent_managed_by_system_hint": "此智能体由服务器环境配置管理,无法进行修改或删除。",
"skills_count_value": "{count} 个技能",
"autofill_agent": "自动填充智能体",
"ai_provider": "AI 提供商",