feat(search): generate query embeddings directly for semantic search (#476)

* feat(search): generate query embeddings directly for semantic search

* fix(search): resolve embedding provider from env, redact query logs, and preserve case in cache
This commit is contained in:
Yiling-J
2026-10-15 20:49:45 +08:00
committed by GitHub
parent b7271777e0
commit 6cc90bf973
18 changed files with 546 additions and 423 deletions
+2
View File
@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Changed
* **search**: Significantly improve semantic search response times by generating query text embeddings directly with in-memory caching instead of dispatching background tasks
## [0.5.2] - 2026-10-13
### Added
-1
View File
@@ -1,4 +1,3 @@
export * from '@shumai/agent/src/workflows/agent-autofill'
export * from '@shumai/agent/src/workflows/agent-chat'
export * from '@shumai/agent/src/workflows/agent-embedding'
export * from '@shumai/agent/src/workflows/query-embedding-for-search'
+4 -3
View File
@@ -151,7 +151,7 @@
"@anthropic-ai/sandbox-runtime": "0.0.74",
"@earendil-works/pi-agent-core": "0.82.1",
"@earendil-works/pi-ai": "0.82.1",
"@google/genai": "2.19.0",
"@google/genai": "2.28.0",
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
@@ -211,6 +211,7 @@
"@aws-sdk/s3-request-presigner": "3.1121.0",
"@earendil-works/pi-agent-core": "0.82.1",
"@earendil-works/pi-ai": "0.82.1",
"@google/genai": "2.28.0",
"@modelcontextprotocol/client": "2.0.0",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
@@ -275,7 +276,7 @@
"version": "0.5.2",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"@google/genai": "2.19.0",
"@google/genai": "2.28.0",
"@shumai/agent": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -653,7 +654,7 @@
"@floating-ui/utils": ["@floating-ui/utils@0.2.12", "", {}, "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww=="],
"@google/genai": ["@google/genai@2.19.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-IM3u1FjpOLyOPsPh0smGA6Ntj2olUWCPbTs/WQj7cA/2AKCGw/652W7IaySx0XtwuscFnMYBoraebQusSZ0zSQ=="],
"@google/genai": ["@google/genai@2.28.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-hyNAJj+ywu3L46qlZod6KO/JvcT3QONz6VCiPevC/FWOjAD+LQb7DMLUoV8vzoLpt61Q0ruQX99iqbxEjPJWmw=="],
"@grpc/grpc-js": ["@grpc/grpc-js@1.14.4", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ=="],
+1 -1
View File
@@ -9,7 +9,7 @@
"@anthropic-ai/sandbox-runtime": "0.0.74",
"@earendil-works/pi-agent-core": "0.82.1",
"@earendil-works/pi-ai": "0.82.1",
"@google/genai": "2.19.0",
"@google/genai": "2.28.0",
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
+23 -34
View File
@@ -105,6 +105,8 @@ async function callGemmaEmbeddingServer(input: string | OpenAiEmbeddingItem[]):
}
}
import { textEmbeddingService } from '@shumai/core/src/search/text-embedding'
export interface GenerateTextEmbeddingParams {
text: string
teamId: string
@@ -115,48 +117,35 @@ export async function generateTextEmbeddingActivity(params: GenerateTextEmbeddin
usage: Usage
}> {
const provider = process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
if (provider === 'gemma-embedding-2-openai') {
return callGemmaEmbeddingServer(`task: search query | text: ${params.text}`)
}
const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
if (provider !== 'gemma-embedding-2-openai' && !process.env.GEMINI_API_KEY) {
throw ApplicationFailure.create({
message: 'GEMINI_API_KEY environment variable is not configured',
nonRetryable: true,
})
}
const resolvedApiKey = process.env[apiKey] || apiKey
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
try {
const result = await textEmbeddingService.generateEmbedding(params.text, {
teamId: params.teamId,
skipCache: true,
})
const usage: Usage = {
model: 'gemini-embedding-2',
inputTokens: 0,
outputTokens: 0,
}
const response = await ai.models.embedContent({
model: 'gemini-embedding-2',
contents: [
{
parts: [{ text: params.text }],
role: 'user',
return {
embedding: result.embedding,
usage: {
model: result.usage?.model || provider,
inputTokens: result.usage?.inputTokens ?? 0,
outputTokens: result.usage?.outputTokens ?? 0,
},
],
config: {
outputDimensionality: 1536,
},
})
const values = response.embeddings?.[0]?.values
if (!values) {
throw new Error('No embedding values returned from gemini-embedding-2')
}
return {
embedding: values,
usage,
}
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
throw ApplicationFailure.create({
message: err instanceof Error ? err.message : String(err),
nonRetryable: false,
})
}
}
-3
View File
@@ -610,11 +610,9 @@ import * as aiActivities from './activities/ai'
import { agentAutofillMedia } from './workflows/agent-autofill'
import { agentChat } from './workflows/agent-chat'
import { agentEmbeddingMedia } from './workflows/agent-embedding'
import { queryEmbeddingForSearch } from './workflows/query-embedding-for-search'
export function initAgentWorkflows() {
registerWorkflow(WorkflowTaskType.ai_embedding, agentEmbeddingMedia)
registerWorkflow(WorkflowTaskType.query_embedding_for_search, queryEmbeddingForSearch)
registerWorkflow(WorkflowTaskType.ai_metadata_autofill, agentAutofillMedia)
registerWorkflow(WorkflowTaskType.chat, agentChat)
@@ -637,7 +635,6 @@ export * from './context/serialize-context'
export * from './workflows/agent-autofill'
export * from './workflows/agent-chat'
export * from './workflows/agent-embedding'
export * from './workflows/query-embedding-for-search'
export * from './tools/generate-image'
export * from './tools/generate-video'
export * from './tools/rename-asset'
@@ -1,145 +0,0 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { queryEmbeddingForSearch } from './query-embedding-for-search'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import * as workflowUtils from '@shumai/workflow-core'
vi.mock('@shumai/workflow-core', async () => {
const actual = await vi.importActual('@shumai/workflow-core')
return {
...actual,
getActivities: vi.fn(),
executeActivity: vi.fn(),
}
})
describe('Query Embedding For Search Workflow', () => {
setupTestDbHooks()
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities holds mock functions cast to expected types
let mockActivities: any
beforeEach(() => {
vi.clearAllMocks()
mockActivities = {
updateWorkflowTaskActivity: Object.assign(vi.fn(), {
_activityName: 'updateWorkflowTaskActivity',
}),
generateTextEmbeddingActivity: Object.assign(vi.fn(), {
_activityName: 'generateTextEmbeddingActivity',
}),
getAgentWorkerQueueActivity: Object.assign(vi.fn(), {
_activityName: 'getAgentWorkerQueueActivity',
}),
}
mockActivities.getAgentWorkerQueueActivity.mockResolvedValue('agent_queue')
mockActivities.updateWorkflowTaskActivity.mockResolvedValue({})
mockActivities.generateTextEmbeddingActivity.mockResolvedValue({
embedding: [0.1, 0.2, 0.3],
usage: { inputTokens: 4, outputTokens: 0, model: 'gemini-embedding-2' },
})
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- mockActivities contains vi.fn mock functions which are cast to expected activity proxy types
vi.mocked(workflowUtils.getActivities).mockReturnValue(mockActivities as any)
vi.mocked(workflowUtils.executeActivity).mockImplementation(async (_queue, act, ...args) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- act is one of the mocked activities
return (act as any)(...args)
})
})
it('should run query embedding workflow successfully', async () => {
const task = await prisma.workflowTask.create({
data: {
type: 'query_embedding_for_search',
status: 'pending',
teamId: 't1',
assetId: 'a1',
payload: {
projectId: 'p1',
queryEmbeddingForSearch: {
text: 'search query',
},
},
},
})
await queryEmbeddingForSearch(task)
// Verify queue discovery
expect(mockActivities.getAgentWorkerQueueActivity).toHaveBeenCalled()
// Verify task processing status
expect(mockActivities.updateWorkflowTaskActivity).toHaveBeenCalledWith({
taskId: task.id,
status: 'processing',
heartbeat: true,
})
// Verify embedding generation
expect(mockActivities.generateTextEmbeddingActivity).toHaveBeenCalledWith({
text: 'search query',
teamId: 't1',
})
// Verify completed status and details
expect(mockActivities.updateWorkflowTaskActivity).toHaveBeenCalledWith({
taskId: task.id,
status: 'completed',
output: { embedding: [0.1, 0.2, 0.3] },
inputTokens: 4,
outputTokens: 0,
model: 'gemini-embedding-2',
})
})
it('should throw immediately if text is missing', async () => {
const task = await prisma.workflowTask.create({
data: {
type: 'query_embedding_for_search',
status: 'pending',
teamId: 't1',
assetId: 'a1',
},
})
await expect(queryEmbeddingForSearch(task)).rejects.toThrow(
'Missing text in query_embedding_for_search task payload',
)
// Should not call any activities because it throws before queue discovery
expect(mockActivities.getAgentWorkerQueueActivity).not.toHaveBeenCalled()
expect(mockActivities.updateWorkflowTaskActivity).not.toHaveBeenCalled()
})
it('should handle activity failure and update task status to failed', async () => {
mockActivities.generateTextEmbeddingActivity.mockRejectedValue(
new Error('Embedding API failed'),
)
const task = await prisma.workflowTask.create({
data: {
type: 'query_embedding_for_search',
status: 'pending',
teamId: 't1',
assetId: 'a1',
payload: {
projectId: 'p1',
queryEmbeddingForSearch: {
text: 'search query',
},
},
},
})
await expect(queryEmbeddingForSearch(task)).rejects.toThrow('Embedding API failed')
// Verify it updates task to failed
expect(mockActivities.updateWorkflowTaskActivity).toHaveBeenCalledWith({
taskId: task.id,
status: 'failed',
output: { error: 'Embedding API failed' },
})
})
})
@@ -1,56 +0,0 @@
import type { WorkflowTask } from '@shumai/db'
import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core'
export async function queryEmbeddingForSearch(task: WorkflowTask): Promise<void> {
const { generateTextEmbeddingActivity, updateWorkflowTaskActivity, getAgentWorkerQueueActivity } =
getActivities()
let agentWorkerQueue = ''
const payload = task.payload as PrismaJson.WorkflowTaskPayload | null
const text = payload?.queryEmbeddingForSearch?.text
if (!text) {
throw new Error('Missing text in query_embedding_for_search task payload')
}
try {
// 0. Discover queue
agentWorkerQueue = await executeActivity(TaskQueueAgent, getAgentWorkerQueueActivity)
// 1. Mark as processing
await executeActivity(agentWorkerQueue, updateWorkflowTaskActivity, {
taskId: task.id,
status: 'processing',
heartbeat: true,
})
// 2. Generate embedding
const result = await executeActivity(agentWorkerQueue, generateTextEmbeddingActivity, {
text,
teamId: task.teamId!,
})
// 3. Save output and complete
await executeActivity(agentWorkerQueue, updateWorkflowTaskActivity, {
taskId: task.id,
status: 'completed',
output: { embedding: result.embedding },
inputTokens: result.usage.inputTokens,
outputTokens: result.usage.outputTokens,
model: result.usage.model,
})
} catch (err) {
console.error(`queryEmbeddingForSearch failed for task ${task.id}:`, err)
if (agentWorkerQueue) {
await executeActivity(agentWorkerQueue, updateWorkflowTaskActivity, {
taskId: task.id,
status: 'failed',
output: {
error: err instanceof Error ? err.message : String(err),
},
})
}
throw err
}
}
+1
View File
@@ -29,6 +29,7 @@
"@aws-sdk/s3-request-presigner": "3.1121.0",
"@earendil-works/pi-agent-core": "0.82.1",
"@earendil-works/pi-ai": "0.82.1",
"@google/genai": "2.28.0",
"@modelcontextprotocol/client": "2.0.0",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
+1
View File
@@ -10,6 +10,7 @@ export * from './project/project'
export * from './provider/provider'
export * from './s3/s3'
export * from './search/search'
export * from './search/text-embedding'
export * from './share/share'
export * from './skill/skill'
export * from './team/team'
+62 -24
View File
@@ -4,7 +4,7 @@ import { setupTestDbHooks } from '@shumai/db/test'
import { SearchService } from './search'
import { SearchConditionOperator } from '@shumai/dtos'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { workflowService } from '@shumai/workflow-core'
import { textEmbeddingService } from './text-embedding'
import { AssetType } from '@shumai/db'
import { assetService } from '@shumai/core/src/asset/asset'
@@ -805,20 +805,14 @@ describe('SearchService', () => {
)
`
// Mock the embedding generation workflow task.
// Mock the embedding generation
// Our query vector points along the first dimension [1, 0, 0...]
const queryVector = Array(1536).fill(0)
queryVector[0] = 1.0
const mockExecuteWait = vi
.spyOn(workflowService, 'executeWait')
.mockImplementation(async (task) => {
return {
...task,
status: 'completed',
output: {
embedding: queryVector,
},
} as unknown as typeof task
const mockGenerateEmbedding = vi
.spyOn(textEmbeddingService, 'generateEmbedding')
.mockResolvedValue({
embedding: queryVector,
})
// Search using semantic mode
@@ -855,7 +849,7 @@ describe('SearchService', () => {
expect(result.data[2].startTime).toBe(10)
expect(result.data[2].endTime).toBe(20)
mockExecuteWait.mockRestore()
mockGenerateEmbedding.mockRestore()
})
it('matches and sorts 768-dimensional embeddings using embedding_768 column', async () => {
@@ -978,16 +972,10 @@ describe('SearchService', () => {
// Query vector with 768 dims pointing along dimension 0
const queryVector = Array(768).fill(0)
queryVector[0] = 1.0
const mockExecuteWait = vi
.spyOn(workflowService, 'executeWait')
.mockImplementation(async (task) => {
return {
...task,
status: 'completed',
output: {
embedding: queryVector,
},
} as unknown as typeof task
const mockGenerateEmbedding = vi
.spyOn(textEmbeddingService, 'generateEmbedding')
.mockResolvedValue({
embedding: queryVector,
})
const result = await searchService.search(rootFolder.id, {
@@ -1003,7 +991,57 @@ describe('SearchService', () => {
expect(result.data[0].id).toBe(videoAssetA.id)
expect(result.data[1].id).toBe(videoAssetB.id)
mockExecuteWait.mockRestore()
mockGenerateEmbedding.mockRestore()
})
it('throws 502 if embedding generation fails during semantic search', async () => {
const user = await prisma.user.create({
data: { name: 'semantic-user-fail', email: 'semantic-fail@example.com', type: 'human' },
})
const team = await prisma.team.create({
data: { name: 'semantic-team-fail' },
})
const project = await prisma.project.create({
data: { name: 'semantic-project-fail', teamId: team.id },
})
const botUser = await prisma.user.create({
data: { name: 'Embedding Bot Fail', email: 'emb-fail@example.com', type: 'agent' },
})
await prisma.agent.create({
data: {
id: botUser.id,
teamId: team.id,
type: 'embedding',
enabled: true,
config: { provider: 'gemini-embedding-2', model: 'gemini-embedding-2' },
},
})
const rootFolder = await prisma.asset.create({
data: {
name: 'root-fail',
type: AssetType.folder,
projectId: project.id,
creatorId: user.id,
status: 'uploaded',
},
})
const mockGenerateEmbedding = vi
.spyOn(textEmbeddingService, 'generateEmbedding')
.mockRejectedValue(new Error('Rate limit exceeded'))
await expect(
searchService.search(rootFolder.id, {
recursively: true,
assetType: 'file',
operator: 'AND',
conditions: [],
query: 'fail query',
isSemantic: true,
}),
).rejects.toThrow('Failed to generate query embedding: Rate limit exceeded')
mockGenerateEmbedding.mockRestore()
})
})
+19 -20
View File
@@ -1,18 +1,19 @@
import { prisma } from '@shumai/db'
import { Prisma, AssetType, WorkflowTaskType } from '@shumai/db'
import { Prisma, AssetType } from '@shumai/db'
import { AssetService, assetService } from '@shumai/core/src/asset/asset'
import { AssetInfo } from '@shumai/dtos'
import { SearchRequest } from '@shumai/dtos'
import { PaginatedData, decodeCursor, encodeCursor, PageInfo } from '@shumai/core/src/pagination'
import { generateSearchNgrams } from '@shumai/core/src/utils/ngram'
import { workflowService } from '@shumai/workflow-core'
import { HTTPException } from 'hono/http-exception'
import { SqlQueryBuilder } from './sql-query-builder'
import { TextEmbeddingService, textEmbeddingService } from './text-embedding'
export class SearchService {
constructor(
private readonly prismaClient: typeof prisma = prisma,
private readonly assetSvc: AssetService = assetService,
private readonly embeddingSvc: TextEmbeddingService = textEmbeddingService,
) {}
async search(folderId: string, req: SearchRequest): Promise<PaginatedData<AssetInfo[]>> {
@@ -44,26 +45,24 @@ export class SearchService {
})
}
// 2. Generate query embedding via workflow
const task = await this.prismaClient.workflowTask.create({
data: {
type: WorkflowTaskType.query_embedding_for_search,
// 2. Generate query embedding directly using configured environment provider
let queryVector: number[]
try {
const result = await this.embeddingSvc.generateEmbedding(req.query, {
teamId,
assetId: folderId,
payload: {
projectId: '',
queryEmbeddingForSearch: { text: req.query },
} as PrismaJson.WorkflowTaskPayload,
status: 'pending',
},
})
})
queryVector = result.embedding
} catch (err) {
const msg = err instanceof Error ? err.message : String(err)
throw new HTTPException(502, {
message: `Failed to generate query embedding: ${msg}`,
})
}
const completedTask = await workflowService.executeWait(task)
const output = completedTask.output as Record<string, unknown> | null
const queryVector = output?.embedding as number[] | undefined
if (!queryVector) {
throw new Error('Failed to generate query embedding')
if (!queryVector || queryVector.length === 0) {
throw new HTTPException(502, {
message: 'Failed to generate query embedding: empty vector returned',
})
}
// 3. Construct raw SQL using SqlQueryBuilder
@@ -0,0 +1,236 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { TextEmbeddingService } from './text-embedding'
import { logger } from '@shumai/core/src/logger'
const mockEmbedContent = vi.fn()
vi.mock('@google/genai', () => {
return {
// eslint-disable-next-line @typescript-eslint/naming-convention
GoogleGenAI: class {
models = {
embedContent: mockEmbedContent,
}
},
}
})
describe('TextEmbeddingService', () => {
const originalEnv = { ...process.env }
let service: TextEmbeddingService
beforeEach(() => {
process.env = { ...originalEnv }
process.env.GEMINI_API_KEY = 'test-gemini-key'
delete process.env.EMBEDDING_PROVIDER
delete process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL
mockEmbedContent.mockReset()
service = new TextEmbeddingService(500, 3600 * 1000)
})
afterEach(() => {
process.env = originalEnv
vi.restoreAllMocks()
})
it('throws an error if text is empty or whitespace', async () => {
await expect(service.generateEmbedding('')).rejects.toThrow('Text to embed must not be empty')
await expect(service.generateEmbedding(' ')).rejects.toThrow(
'Text to embed must not be empty',
)
})
it('generates Gemini embedding (1536d) by default', async () => {
const dummyEmbedding = Array(1536).fill(0.05)
mockEmbedContent.mockResolvedValueOnce({
embeddings: [{ values: dummyEmbedding }],
})
const result = await service.generateEmbedding('beautiful sunset')
expect(result.embedding).toEqual(dummyEmbedding)
expect(result.usage?.model).toBe('gemini-embedding-2')
expect(result.cached).toBeUndefined()
expect(mockEmbedContent).toHaveBeenCalledWith(
expect.objectContaining({
model: 'gemini-embedding-2',
contents: [
expect.objectContaining({
parts: [{ text: 'beautiful sunset' }],
}),
],
config: {
outputDimensionality: 1536,
},
}),
)
})
it('returns cached result on repeated call without re-invoking the provider', async () => {
const dummyEmbedding = Array(1536).fill(0.05)
mockEmbedContent.mockResolvedValueOnce({
embeddings: [{ values: dummyEmbedding }],
})
const first = await service.generateEmbedding('ocean breeze')
expect(first.embedding).toEqual(dummyEmbedding)
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
// Second call with same text and extra whitespace
const second = await service.generateEmbedding(' ocean breeze ')
expect(second.embedding).toEqual(dummyEmbedding)
expect(second.cached).toBe(true)
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
})
it('preserves case distinctions in cache keys', async () => {
const dummyApple = Array(1536).fill(0.1)
const dummyLowerApple = Array(1536).fill(0.2)
mockEmbedContent
.mockResolvedValueOnce({
embeddings: [{ values: dummyApple }],
})
.mockResolvedValueOnce({
embeddings: [{ values: dummyLowerApple }],
})
const upper = await service.generateEmbedding('Apple')
expect(upper.embedding).toEqual(dummyApple)
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
const lower = await service.generateEmbedding('apple')
expect(lower.embedding).toEqual(dummyLowerApple)
expect(lower.cached).toBeUndefined()
expect(mockEmbedContent).toHaveBeenCalledTimes(2)
})
it('does not log raw query text on cache hits', async () => {
const dummyEmbedding = Array(1536).fill(0.05)
mockEmbedContent.mockResolvedValue({
embeddings: [{ values: dummyEmbedding }],
})
const debugSpy = vi.spyOn(logger, 'debug')
await service.generateEmbedding('confidential search query')
await service.generateEmbedding('confidential search query')
expect(debugSpy).toHaveBeenCalledWith(
expect.not.objectContaining({
key: expect.anything(),
text: expect.anything(),
}),
'Cache hit for text embedding',
)
})
it('clears cache when clearCache is called', async () => {
const dummyEmbedding = Array(1536).fill(0.05)
mockEmbedContent.mockResolvedValue({
embeddings: [{ values: dummyEmbedding }],
})
await service.generateEmbedding('query')
expect(mockEmbedContent).toHaveBeenCalledTimes(1)
service.clearCache()
await service.generateEmbedding('query')
expect(mockEmbedContent).toHaveBeenCalledTimes(2)
})
it('throws if GEMINI_API_KEY is not configured', async () => {
delete process.env.GEMINI_API_KEY
await expect(service.generateEmbedding('test')).rejects.toThrow(
'GEMINI_API_KEY environment variable is not configured',
)
})
it('throws if Gemini returns empty embedding values', async () => {
mockEmbedContent.mockResolvedValueOnce({
embeddings: [{ values: null }],
})
await expect(service.generateEmbedding('test')).rejects.toThrow(
'No embedding values returned from gemini-embedding-2',
)
})
it('generates Gemma embedding (768d) when provider is gemma-embedding-2-openai', async () => {
const dummyEmbedding = Array(768).fill(0.12)
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
model: 'embeddinggemma-2',
data: [{ embedding: dummyEmbedding }],
// eslint-disable-next-line @typescript-eslint/naming-convention
usage: { prompt_tokens: 12 },
}),
})
vi.stubGlobal('fetch', mockFetch)
const result = await service.generateEmbedding('drone footage', {
provider: 'gemma-embedding-2-openai',
})
expect(result.embedding).toEqual(dummyEmbedding)
expect(result.usage?.model).toBe('embeddinggemma-2')
expect(result.usage?.inputTokens).toBe(12)
expect(mockFetch).toHaveBeenCalledWith(
'http://localhost:9379/v1/embeddings',
expect.objectContaining({
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: 'embeddinggemma-2',
input: 'task: search query | text: drone footage',
}),
}),
)
})
it('respects GEMMA_EMBEDDING_OPENAI_BASE_URL without /v1 suffix', async () => {
process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL = 'http://custom-host:9379'
const dummyEmbedding = Array(768).fill(0.12)
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
data: [{ embedding: dummyEmbedding }],
}),
})
vi.stubGlobal('fetch', mockFetch)
await service.generateEmbedding('test query', {
provider: 'gemma-embedding-2-openai',
})
expect(mockFetch).toHaveBeenCalledWith(
'http://custom-host:9379/v1/embeddings',
expect.anything(),
)
})
it('throws descriptive error when Gemma server returns non-ok status', async () => {
const mockFetch = vi.fn().mockResolvedValue({
ok: false,
status: 503,
text: async () => 'Service Unavailable',
})
vi.stubGlobal('fetch', mockFetch)
await expect(
service.generateEmbedding('test query', {
provider: 'gemma-embedding-2-openai',
}),
).rejects.toThrow('Embedding server returned status 503: Service Unavailable')
})
it('throws error when abort signal is triggered', async () => {
const controller = new AbortController()
controller.abort(new Error('Operation cancelled by user'))
await expect(
service.generateEmbedding('test query', {
signal: controller.signal,
}),
).rejects.toThrow('Operation cancelled by user')
})
})
+196
View File
@@ -0,0 +1,196 @@
import { GoogleGenAI } from '@google/genai'
import { LruTtlCache } from '@shumai/core/src/cache/lru-ttl-cache'
import { logger } from '@shumai/core/src/logger'
export interface TextEmbeddingUsage {
model: string
inputTokens: number
outputTokens: number
}
export interface TextEmbeddingResult {
embedding: number[]
usage?: TextEmbeddingUsage
cached?: boolean
}
export interface GenerateTextEmbeddingOptions {
provider?: string
model?: string
signal?: AbortSignal
teamId?: string
skipCache?: boolean
}
export class TextEmbeddingService {
private cache: LruTtlCache<string, number[]>
constructor(
cacheCapacity: number = 500,
private defaultTtlMs: number = 3600 * 1000,
) {
this.cache = new LruTtlCache<string, number[]>(cacheCapacity, 50)
}
/**
* Generates a text embedding vector directly using the configured provider.
* If a cached embedding exists for the text, provider, and model, it is returned immediately.
*/
async generateEmbedding(
text: string,
options?: GenerateTextEmbeddingOptions,
): Promise<TextEmbeddingResult> {
const trimmedText = text.trim()
if (!trimmedText) {
throw new Error('Text to embed must not be empty')
}
const provider = options?.provider || process.env.EMBEDDING_PROVIDER || 'gemini-embedding-2'
const model =
options?.model ||
(provider === 'gemma-embedding-2-openai' ? 'embeddinggemma-2' : 'gemini-embedding-2')
const cacheKey = `${provider}:${model}:${trimmedText}`
if (!options?.skipCache) {
const cachedVector = this.cache.get(cacheKey)
if (cachedVector) {
logger.debug({ provider, model }, 'Cache hit for text embedding')
return {
embedding: cachedVector,
usage: { model, inputTokens: 0, outputTokens: 0 },
cached: true,
}
}
}
const signal = options?.signal ?? AbortSignal.timeout(5000)
let result: TextEmbeddingResult
if (provider === 'gemma-embedding-2-openai') {
result = await this.callGemmaEmbeddingServer(trimmedText, model, signal)
} else {
result = await this.callGeminiEmbedding(trimmedText, model, signal)
}
if (!options?.skipCache) {
this.cache.set(cacheKey, result.embedding, this.defaultTtlMs)
}
return result
}
private async callGeminiEmbedding(
text: string,
model: string,
signal: AbortSignal,
): Promise<TextEmbeddingResult> {
const apiKey = process.env.GEMINI_API_KEY
if (!apiKey) {
throw new Error('GEMINI_API_KEY environment variable is not configured')
}
if (signal.aborted) {
throw signal.reason ?? new Error('Embedding generation aborted or timed out')
}
const resolvedApiKey = process.env[apiKey] || apiKey
const ai = new GoogleGenAI({ apiKey: resolvedApiKey })
const embedPromise = ai.models.embedContent({
model,
contents: [
{
parts: [{ text }],
role: 'user',
},
],
config: {
outputDimensionality: 1536,
},
})
const abortPromise = new Promise<never>((_, reject) => {
if (signal.aborted) {
reject(signal.reason ?? new Error('Embedding generation aborted or timed out'))
return
}
signal.addEventListener(
'abort',
() => {
reject(signal.reason ?? new Error('Embedding generation aborted or timed out'))
},
{ once: true },
)
})
const response = await Promise.race([embedPromise, abortPromise])
const values = response.embeddings?.[0]?.values
if (!values || !Array.isArray(values)) {
throw new Error(`No embedding values returned from ${model}`)
}
return {
embedding: values,
usage: {
model,
inputTokens: 0,
outputTokens: 0,
},
}
}
private async callGemmaEmbeddingServer(
text: string,
model: string,
signal: AbortSignal,
): Promise<TextEmbeddingResult> {
const rawBaseUrl = process.env.GEMMA_EMBEDDING_OPENAI_BASE_URL || 'http://localhost:9379/v1'
const normalized = rawBaseUrl.replace(/\/+$/, '')
const endpoint = normalized.endsWith('/embeddings')
? normalized
: normalized.endsWith('/v1')
? `${normalized}/embeddings`
: `${normalized}/v1/embeddings`
const response = await fetch(endpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
input: `task: search query | text: ${text}`,
}),
signal,
})
if (!response.ok) {
const errorText = await response.text()
throw new Error(`Embedding server returned status ${response.status}: ${errorText}`)
}
const json = (await response.json()) as {
model?: string
data?: Array<{ embedding?: number[] }>
// eslint-disable-next-line @typescript-eslint/naming-convention
usage?: { prompt_tokens?: number }
}
const embedding = json.data?.[0]?.embedding
if (!embedding || !Array.isArray(embedding)) {
throw new Error('Invalid embedding returned from embedding server')
}
return {
embedding,
usage: {
model: json.model || model,
inputTokens: json.usage?.prompt_tokens ?? 0,
outputTokens: 0,
},
}
}
clearCache(): void {
this.cache.clear()
}
}
export const textEmbeddingService = new TextEmbeddingService()
+1 -1
View File
@@ -5,7 +5,7 @@
"type": "module",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"@google/genai": "2.19.0",
"@google/genai": "2.28.0",
"@shumai/agent": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -1,127 +0,0 @@
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { workflowService, TaskQueueAgent } from '@shumai/workflow-core'
import { initAgentWorkflows } from '@shumai/agent'
import { s3Service } from '@shumai/core/src/s3/s3'
const mockEmbedContent = vi.fn().mockResolvedValue({
embeddings: [
{
values: Array(1536).fill(0.35),
},
],
})
vi.mock('@google/genai', () => {
return {
// eslint-disable-next-line @typescript-eslint/naming-convention
GoogleGenAI: class {
models = {
embedContent: mockEmbedContent,
}
},
}
})
import { fileURLToPath } from 'url'
import * as path from 'path'
const currentDir = path.dirname(fileURLToPath(import.meta.url))
const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts')
describe.each(['local', 'temporal'] as const)(
'Workflow E2E - queryEmbeddingForSearch (executor: %s)',
(mode) => {
setupTestDbHooks()
let agentWorkerPromise: Promise<void> | null = null
beforeAll(async () => {
process.env.S3_BUCKET = 'shumai-e2e-test-bucket-query'
process.env.GEMINI_API_KEY = 'dummy-key'
workflowService.setExecutorType(mode)
initAgentWorkflows()
if (mode === 'temporal') {
console.log('Starting background worker for query embedding Temporal E2E tests...')
agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, {
workflowsPath: agentWorkflowsPath,
})
await new Promise((resolve) => setTimeout(resolve, 2000))
} else {
console.log('Starting local workflow service polling...')
workflowService.start()
}
})
afterAll(async () => {
if (mode === 'temporal') {
console.log('Shutting down Temporal workers...')
await workflowService.shutdownWorkers()
await Promise.all([agentWorkerPromise].filter(Boolean))
}
workflowService.close()
vi.restoreAllMocks()
try {
console.log('Cleaning up local E2E storage files...')
await s3Service.deletePrefix('shumai-e2e-test-bucket-query', '')
} catch (err) {
console.error('Failed to clean up E2E storage folder:', err)
}
})
it('should run queryEmbeddingForSearch workflow and return embedding successfully', async () => {
// 1. Seed Database
const team = await prisma.team.create({
data: { name: 'E2E Query Embedding Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Query Embedding Project', teamId: team.id },
})
const asset = await prisma.asset.create({
data: {
name: 'dummy-search-asset',
type: 'file',
status: 'uploaded',
mediaType: 'text/plain',
projectId: project.id,
},
})
// 2. Create Workflow Task
const task = await prisma.workflowTask.create({
data: {
type: 'query_embedding_for_search',
status: 'pending',
teamId: team.id,
projectId: project.id,
assetId: asset.id,
payload: {
projectId: project.id,
queryEmbeddingForSearch: {
text: 'test search query',
},
},
},
})
// 3. Wait for workflow to complete
console.log(
`Submitted E2E Query Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`,
)
const completedTask = await workflowService.executeWait(task, 45000)
// 4. Verification
expect(completedTask.status).toBe('completed')
// Verify output payload contains correct embedding values and usage metadata
const output = completedTask.output as unknown as { embedding: number[] }
expect(output).toBeDefined()
expect(output.embedding).toEqual(Array(1536).fill(0.35))
expect(completedTask.model).toBe('gemini-embedding-2')
}, 50000)
},
)
@@ -8,7 +8,6 @@ import { WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db'
// Mock the individual workflows
const mocks = {
agentEmbeddingMedia: vi.fn(),
queryEmbeddingForSearch: vi.fn(),
agentAutofillMedia: vi.fn(),
agentChat: vi.fn(),
transcodeMedia: vi.fn(),
@@ -26,9 +25,6 @@ describe('LocalExecutor Integration Tests', () => {
// Register workflows dynamically for testing
registerWorkflow(WorkflowTaskType.ai_embedding, async (task) => mocks.agentEmbeddingMedia(task))
registerWorkflow(WorkflowTaskType.query_embedding_for_search, async (task) =>
mocks.queryEmbeddingForSearch(task),
)
registerWorkflow(WorkflowTaskType.ai_metadata_autofill, async (task) =>
mocks.agentAutofillMedia(task),
)
@@ -22,10 +22,6 @@ export class TemporalExecutor implements Executor {
workflowName = 'agentEmbeddingWorkflow'
taskQueue = TaskQueueAgent
break
case WorkflowTaskType.query_embedding_for_search:
workflowName = 'queryEmbeddingForSearch'
taskQueue = TaskQueueAgent
break
case WorkflowTaskType.ai_metadata_autofill:
workflowName = 'agentAutofillWorkflow'
taskQueue = TaskQueueAgent