diff --git a/bun.lock b/bun.lock index 1d06ec5f..65d4b2ed 100644 --- a/bun.lock +++ b/bun.lock @@ -189,6 +189,7 @@ "version": "0.1.9", "dependencies": { "@earendil-works/pi-agent-core": "0.78.0", + "@google/genai": "1.52.0", "@shumai/agent": "workspace:*", "@shumai/core": "workspace:*", "@shumai/db": "workspace:*", diff --git a/packages/db/src/prisma-testing-helper.ts b/packages/db/src/prisma-testing-helper.ts index b2815e81..77deda33 100644 --- a/packages/db/src/prisma-testing-helper.ts +++ b/packages/db/src/prisma-testing-helper.ts @@ -235,7 +235,7 @@ export class PrismaTestingHelper< }) throw internalRollbackErrorSymbol }, - { ...opts, timeout: 30000 }, + { timeout: 60000, ...opts }, ) .catch((error) => { if (error !== internalRollbackErrorSymbol) { diff --git a/packages/e2e/fixtures/small.mp4 b/packages/e2e/fixtures/small.mp4 new file mode 100644 index 00000000..e509d947 Binary files /dev/null and b/packages/e2e/fixtures/small.mp4 differ diff --git a/packages/e2e/fixtures/small.png b/packages/e2e/fixtures/small.png new file mode 100644 index 00000000..909c66db Binary files /dev/null and b/packages/e2e/fixtures/small.png differ diff --git a/packages/e2e/fixtures/small.wav b/packages/e2e/fixtures/small.wav new file mode 100644 index 00000000..15cf6c45 Binary files /dev/null and b/packages/e2e/fixtures/small.wav differ diff --git a/packages/e2e/package.json b/packages/e2e/package.json index 104b50c6..f4c56e08 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -10,6 +10,7 @@ "@shumai/agent": "workspace:*", "@shumai/transcode": "workspace:*", "@shumai/dtos": "workspace:*", - "@earendil-works/pi-agent-core": "0.78.0" + "@earendil-works/pi-agent-core": "0.78.0", + "@google/genai": "1.52.0" } } diff --git a/packages/e2e/vitest.config.ts b/packages/e2e/vitest.config.ts index 9c4ccd6b..b45b9653 100644 --- a/packages/e2e/vitest.config.ts +++ b/packages/e2e/vitest.config.ts @@ -7,6 +7,10 @@ export default defineConfig({ globalSetup: './packages/e2e/workflow/test-global-setup.ts', globals: false, pool: 'forks', + fileParallelism: false, + sequence: { + concurrent: false, + }, include: ['packages/e2e/workflow/**/*.test.ts'], server: { deps: { diff --git a/packages/e2e/workflow/agent-autofill.test.ts b/packages/e2e/workflow/agent-autofill.test.ts index 5a839cb2..91e78390 100644 --- a/packages/e2e/workflow/agent-autofill.test.ts +++ b/packages/e2e/workflow/agent-autofill.test.ts @@ -13,319 +13,322 @@ const currentDir = path.dirname(fileURLToPath(import.meta.url)) const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts') const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts') -describe.each(['local', 'temporal'] as const)('Workflow E2E - agentAutofillMedia (executor: %s)', (mode) => { - setupTestDbHooks() +describe.each(['local', 'temporal'] as const)( + 'Workflow E2E - agentAutofillMedia (executor: %s)', + (mode) => { + setupTestDbHooks() - let agentWorkerPromise: Promise | null = null - let transcodeWorkerPromise: Promise | null = null + let agentWorkerPromise: Promise | null = null + let transcodeWorkerPromise: Promise | null = null - beforeAll(async () => { - // Set bucket environment - process.env.S3_BUCKET = 'shumai-e2e-test-bucket' + beforeAll(async () => { + // Set bucket environment + process.env.S3_BUCKET = 'shumai-e2e-test-bucket' - // Configure the workflow service executor dynamically - workflowService.setExecutorType(mode) + // Configure the workflow service executor dynamically + workflowService.setExecutorType(mode) - // Initialize registries - initAgentWorkflows() - initTranscodeWorkflows() + // Initialize registries + initAgentWorkflows() + initTranscodeWorkflows() - // Spy on the harness prompt method to intercept the LLM call while running real activities & tools - vi.spyOn(AgentHarness.prototype, 'prompt').mockImplementation(async function ( - this: AgentHarness, - ) { - // Find the real autofill_metadata tool configured on the harness - const tools = this.getTools() - const autofillTool = tools.find((t: AgentTool) => t.name === 'autofill_metadata') + // Spy on the harness prompt method to intercept the LLM call while running real activities & tools + vi.spyOn(AgentHarness.prototype, 'prompt').mockImplementation(async function ( + this: AgentHarness, + ) { + // Find the real autofill_metadata tool configured on the harness + const tools = this.getTools() + const autofillTool = tools.find((t: AgentTool) => t.name === 'autofill_metadata') - if (autofillTool) { - // Execute the real tool callback to perform the actual DB updates - await autofillTool.execute('call-123', { - title: 'E2E Title Extracted', - confidence: 0.95, - completed: true, + if (autofillTool) { + // Execute the real tool callback to perform the actual DB updates + await autofillTool.execute('call-123', { + title: 'E2E Title Extracted', + confidence: 0.95, + completed: true, + }) + } + + // Return a successful assistant response mock + /* eslint-disable @typescript-eslint/no-explicit-any -- Mock assistant response structure */ + return { + content: [{ type: 'text', text: 'Success' }], + usage: { input: 10, output: 20 }, + stopReason: 'stop', + } as any + /* eslint-enable @typescript-eslint/no-explicit-any */ + }) + + if (mode === 'temporal') { + console.log('Starting background workers for Temporal E2E tests...') + agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, { + workflowsPath: agentWorkflowsPath, }) + transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, { + workflowsPath: transcodeWorkflowsPath, + }) + + // Wait a moment for workers to establish connections + await new Promise((resolve) => setTimeout(resolve, 2000)) + } else { + console.log('Starting local workflow service polling...') + workflowService.start() + } + }) + + afterAll(async () => { + if (mode === 'temporal') { + console.log('Shutting down Temporal workers...') + await workflowService.shutdownWorkers() + await Promise.all([agentWorkerPromise, transcodeWorkerPromise].filter(Boolean)) } - // Return a successful assistant response mock - /* eslint-disable @typescript-eslint/no-explicit-any -- Mock assistant response structure */ - return { - content: [{ type: 'text', text: 'Success' }], - usage: { input: 10, output: 20 }, - stopReason: 'stop', - } as any - /* eslint-enable @typescript-eslint/no-explicit-any */ + workflowService.close() + + // Restore spied methods + vi.restoreAllMocks() + + // Clean up E2E files in local filesystem storage + try { + console.log('Cleaning up local E2E storage files...') + await s3Service.deletePrefix('shumai-e2e-test-bucket', '') + } catch (err) { + console.error('Failed to clean up E2E storage folder:', err) + } }) - if (mode === 'temporal') { - console.log('Starting background workers for Temporal E2E tests...') - agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, { - workflowsPath: agentWorkflowsPath, - }) - transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, { - workflowsPath: transcodeWorkflowsPath, + it('should run agentAutofillMedia workflow and update asset metadata successfully', async () => { + // 1. Seed Database Models + const team = await prisma.team.create({ + data: { name: 'E2E Test Team' }, }) - // Wait a moment for workers to establish connections - await new Promise((resolve) => setTimeout(resolve, 2000)) - } else { - console.log('Starting local workflow service polling...') - workflowService.start() - } - }) + const project = await prisma.project.create({ + data: { name: 'E2E Test Project', teamId: team.id }, + }) - afterAll(async () => { - if (mode === 'temporal') { - console.log('Shutting down Temporal workers...') - await workflowService.shutdownWorkers() - await Promise.all([agentWorkerPromise, transcodeWorkerPromise].filter(Boolean)) - } - - workflowService.close() - - // Restore spied methods - vi.restoreAllMocks() - - // Clean up E2E files in local filesystem storage - try { - console.log('Cleaning up local E2E storage files...') - await s3Service.deletePrefix('shumai-e2e-test-bucket', '') - } catch (err) { - console.error('Failed to clean up E2E storage folder:', err) - } - }) - - it('should run agentAutofillMedia workflow and update asset metadata successfully', async () => { - // 1. Seed Database Models - const team = await prisma.team.create({ - data: { name: 'E2E Test Team' }, - }) - - const project = await prisma.project.create({ - data: { name: 'E2E Test Project', teamId: team.id }, - }) - - // Create Metadata Field (aiAutofill = true, text) - await prisma.metadataField.create({ - data: { - key: 'title', - scope: 'PROJECT', - projectId: project.id, - teamId: team.id, - config: { name: 'Title', type: 'text' }, - aiAutofill: true, - description: 'Auto-extracted title', - }, - }) - - // Create Metadata Field (aiAutofill = true, number) - await prisma.metadataField.create({ - data: { - key: 'confidence', - scope: 'PROJECT', - projectId: project.id, - teamId: team.id, - config: { name: 'Confidence', type: 'number' }, - aiAutofill: true, - description: 'AI extraction confidence', - }, - }) - - // Create Metadata Field (aiAutofill = true, boolean) - await prisma.metadataField.create({ - data: { - key: 'completed', - scope: 'PROJECT', - projectId: project.id, - teamId: team.id, - config: { name: 'Completed', type: 'toggle' }, - aiAutofill: true, - description: 'Completed flag', - }, - }) - - // Create Metadata Field (aiAutofill = false, text) - await prisma.metadataField.create({ - data: { - key: 'manual_notes', - scope: 'PROJECT', - projectId: project.id, - teamId: team.id, - config: { name: 'Manual Notes', type: 'text' }, - aiAutofill: false, - description: 'Manual notes (should not be auto-filled)', - }, - }) - - // Create Agent User - const agentUser = await prisma.user.create({ - data: { - name: 'E2E Autofill Agent User', - email: 'e2e-autofill-agent@shumai.ai', - type: 'agent', - }, - }) - - // Link Agent User to Team - await prisma.teamMember.create({ - data: { - teamId: team.id, - userId: agentUser.id, - role: 'editor', - }, - }) - - // Setup Agent Provider & Model configuration - const provider = await prisma.provider.create({ - data: { - name: 'google', - teamId: team.id, - config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, - }, - }) - - const model = await prisma.model.create({ - data: { - modelId: 'gemini', - name: 'Gemini', - providerId: provider.id, - config: { - input: ['text', 'image'], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 8192, - maxTokens: 2048, - reasoning: false, - }, - }, - }) - - const agent = await prisma.agent.create({ - data: { - id: agentUser.id, - teamId: team.id, - type: 'autofill', - enabled: true, - providerId: provider.id, - modelId: model.id, - config: { - provider: 'google', - model: 'gemini', - }, - }, - }) - - // Create StorageKey first - const storageKey = await prisma.storageKey.create({ - data: { - key: 'projects/e2e/test-image.png', - }, - }) - - // Create Asset - const asset = await prisma.asset.create({ - data: { - name: 'test-image.png', - type: 'file', - status: 'uploaded', - mediaType: 'image/png', - projectId: project.id, - storageKeyId: storageKey.id, - }, - }) - - // 2. Seed S3 Storage (LocalStorageService) with a valid 1x1 base64-encoded PNG image - const base64Png = - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=' - const pngBuffer = Buffer.from(base64Png, 'base64') - await s3Service.putObject( - 'shumai-e2e-test-bucket', - 'projects/e2e/test-image.png', - pngBuffer, - pngBuffer.length, - 'image/png', - ) - - // 3. Create Workflow Task (ai_metadata_autofill, pending) - const task = await prisma.workflowTask.create({ - data: { - type: 'ai_metadata_autofill', - status: 'pending', - assetId: asset.id, - projectId: project.id, - teamId: team.id, - payload: { + // Create Metadata Field (aiAutofill = true, text) + await prisma.metadataField.create({ + data: { + key: 'title', + scope: 'PROJECT', projectId: project.id, - agent: { sessionId: 'mock-session-123', agentId: agent.id }, + teamId: team.id, + config: { name: 'Title', type: 'text' }, + aiAutofill: true, + description: 'Auto-extracted title', }, - }, - }) + }) - // 4. Wait for workflow to complete - console.log(`Submitted E2E Workflow Task. ID: ${task.id}. Awaiting completion...`) - const completedTask = await workflowService.executeWait(task, 45000) + // Create Metadata Field (aiAutofill = true, number) + await prisma.metadataField.create({ + data: { + key: 'confidence', + scope: 'PROJECT', + projectId: project.id, + teamId: team.id, + config: { name: 'Confidence', type: 'number' }, + aiAutofill: true, + description: 'AI extraction confidence', + }, + }) - // 5. Verification - expect(completedTask.status).toBe('completed') + // Create Metadata Field (aiAutofill = true, boolean) + await prisma.metadataField.create({ + data: { + key: 'completed', + scope: 'PROJECT', + projectId: project.id, + teamId: team.id, + config: { name: 'Completed', type: 'toggle' }, + aiAutofill: true, + description: 'Completed flag', + }, + }) - // Verify placeholder comment is updated correctly - const comments = await prisma.assetComment.findMany({ - where: { assetId: asset.id }, - orderBy: { createdAt: 'asc' }, - }) - expect(comments.length).toBeGreaterThan(0) - // The last updated message of placeholder comment should indicate success - const finalComment = comments[comments.length - 1] - expect(finalComment.message).toBe('Autofill completed successfully.') + // Create Metadata Field (aiAutofill = false, text) + await prisma.metadataField.create({ + data: { + key: 'manual_notes', + scope: 'PROJECT', + projectId: project.id, + teamId: team.id, + config: { name: 'Manual Notes', type: 'text' }, + aiAutofill: false, + description: 'Manual notes (should not be auto-filled)', + }, + }) - // Verify AssetMetadataValue contains the AI-extracted title - const titleVal = await prisma.assetMetadataValue.findUnique({ - where: { - // eslint-disable-next-line @typescript-eslint/naming-convention - assetId_fieldKey: { + // Create Agent User + const agentUser = await prisma.user.create({ + data: { + name: 'E2E Autofill Agent User', + email: 'e2e-autofill-agent@shumai.ai', + type: 'agent', + }, + }) + + // Link Agent User to Team + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: agentUser.id, + role: 'editor', + }, + }) + + // Setup Agent Provider & Model configuration + const provider = await prisma.provider.create({ + data: { + name: 'google', + teamId: team.id, + config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, + }, + }) + + const model = await prisma.model.create({ + data: { + modelId: 'gemini', + name: 'Gemini', + providerId: provider.id, + config: { + input: ['text', 'image'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + reasoning: false, + }, + }, + }) + + const agent = await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'autofill', + enabled: true, + providerId: provider.id, + modelId: model.id, + config: { + provider: 'google', + model: 'gemini', + }, + }, + }) + + // Create StorageKey first + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/test-image.png', + }, + }) + + // Create Asset + const asset = await prisma.asset.create({ + data: { + name: 'test-image.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage (LocalStorageService) with a valid 1x1 base64-encoded PNG image + const base64Png = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=' + const pngBuffer = Buffer.from(base64Png, 'base64') + await s3Service.putObject( + 'shumai-e2e-test-bucket', + 'projects/e2e/test-image.png', + pngBuffer, + pngBuffer.length, + 'image/png', + ) + + // 3. Create Workflow Task (ai_metadata_autofill, pending) + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_metadata_autofill', + status: 'pending', assetId: asset.id, - fieldKey: 'title', + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agent: { sessionId: 'mock-session-123', agentId: agent.id }, + }, }, - }, - }) - expect(titleVal).toBeDefined() - expect(titleVal?.stringValue).toBe('E2E Title Extracted') + }) - // Verify AssetMetadataValue contains the AI-extracted confidence - const confidenceVal = await prisma.assetMetadataValue.findUnique({ - where: { - // eslint-disable-next-line @typescript-eslint/naming-convention - assetId_fieldKey: { - assetId: asset.id, - fieldKey: 'confidence', - }, - }, - }) - expect(confidenceVal).toBeDefined() - expect(confidenceVal?.numberValue).toBe(0.95) + // 4. Wait for workflow to complete + console.log(`Submitted E2E Workflow Task. ID: ${task.id}. Awaiting completion...`) + const completedTask = await workflowService.executeWait(task, 45000) - // Verify AssetMetadataValue contains the AI-extracted completed flag - const completedVal = await prisma.assetMetadataValue.findUnique({ - where: { - // eslint-disable-next-line @typescript-eslint/naming-convention - assetId_fieldKey: { - assetId: asset.id, - fieldKey: 'completed', - }, - }, - }) - expect(completedVal).toBeDefined() - expect(completedVal?.booleanValue).toBe(true) + // 5. Verification + expect(completedTask.status).toBe('completed') - // Verify manual_notes is NOT auto-filled (remains null/undefined in DB) - const manualNotesVal = await prisma.assetMetadataValue.findUnique({ - where: { - // eslint-disable-next-line @typescript-eslint/naming-convention - assetId_fieldKey: { - assetId: asset.id, - fieldKey: 'manual_notes', + // Verify placeholder comment is updated correctly + const comments = await prisma.assetComment.findMany({ + where: { assetId: asset.id }, + orderBy: { createdAt: 'asc' }, + }) + expect(comments.length).toBeGreaterThan(0) + // The last updated message of placeholder comment should indicate success + const finalComment = comments[comments.length - 1] + expect(finalComment.message).toBe('Autofill completed successfully.') + + // Verify AssetMetadataValue contains the AI-extracted title + const titleVal = await prisma.assetMetadataValue.findUnique({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + assetId_fieldKey: { + assetId: asset.id, + fieldKey: 'title', + }, }, - }, - }) - expect(manualNotesVal).toBeNull() - }, 50000) -}) + }) + expect(titleVal).toBeDefined() + expect(titleVal?.stringValue).toBe('E2E Title Extracted') + + // Verify AssetMetadataValue contains the AI-extracted confidence + const confidenceVal = await prisma.assetMetadataValue.findUnique({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + assetId_fieldKey: { + assetId: asset.id, + fieldKey: 'confidence', + }, + }, + }) + expect(confidenceVal).toBeDefined() + expect(confidenceVal?.numberValue).toBe(0.95) + + // Verify AssetMetadataValue contains the AI-extracted completed flag + const completedVal = await prisma.assetMetadataValue.findUnique({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + assetId_fieldKey: { + assetId: asset.id, + fieldKey: 'completed', + }, + }, + }) + expect(completedVal).toBeDefined() + expect(completedVal?.booleanValue).toBe(true) + + // Verify manual_notes is NOT auto-filled (remains null/undefined in DB) + const manualNotesVal = await prisma.assetMetadataValue.findUnique({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + assetId_fieldKey: { + assetId: asset.id, + fieldKey: 'manual_notes', + }, + }, + }) + expect(manualNotesVal).toBeNull() + }, 50000) + }, +) diff --git a/packages/e2e/workflow/agent-chat.test.ts b/packages/e2e/workflow/agent-chat.test.ts new file mode 100644 index 00000000..10669c2c --- /dev/null +++ b/packages/e2e/workflow/agent-chat.test.ts @@ -0,0 +1,328 @@ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { prisma } from '@shumai/db' +import { setupTestDbHooks } from '@shumai/db/test' +import { workflowService, TaskQueueAgent } from '@shumai/workflow-core' +import { initAgentWorkflows } from '@shumai/agent' +import { s3Service } from '@shumai/core/src/s3/s3' +import { AgentHarness } from '@earendil-works/pi-agent-core' +import { fileURLToPath } from 'url' +import * as path from 'path' + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) +const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts') + +describe.each(['local', 'temporal'] as const)('Workflow E2E - agentChat (executor: %s)', (mode) => { + setupTestDbHooks() + + let agentWorkerPromise: Promise | null = null + + beforeAll(async () => { + process.env.S3_BUCKET = 'shumai-e2e-test-bucket-chat' + process.env.GEMINI_API_KEY = 'dummy-key' + + workflowService.setExecutorType(mode) + initAgentWorkflows() + + vi.spyOn(AgentHarness.prototype, 'prompt').mockResolvedValue({ + content: [{ type: 'text', text: 'E2E Agent Chat Response Success' }], + usage: { input: 10, output: 20 }, + stopReason: 'stop', + } as unknown as Awaited>) + + if (mode === 'temporal') { + console.log('Starting background worker for chat Temporal E2E tests...') + agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, { + workflowsPath: agentWorkflowsPath, + }) + await new Promise((resolve) => setTimeout(resolve, 2000)) + } else { + console.log('Starting local workflow service polling...') + workflowService.start() + } + }) + + afterAll(async () => { + if (mode === 'temporal') { + console.log('Shutting down Temporal workers...') + await workflowService.shutdownWorkers() + await Promise.all([agentWorkerPromise].filter(Boolean)) + } + workflowService.close() + vi.restoreAllMocks() + try { + console.log('Cleaning up local E2E storage files...') + await s3Service.deletePrefix('shumai-e2e-test-bucket-chat', '') + } catch (err) { + console.error('Failed to clean up E2E storage folder:', err) + } + }) + + it('should run agentChat workflow with userCommentId and reply comment successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Chat Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Chat Project', teamId: team.id }, + }) + + const agentUser = await prisma.user.create({ + data: { + name: 'E2E Chat Agent User', + email: 'e2e-chat-agent@shumai.ai', + type: 'agent', + }, + }) + + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: agentUser.id, + role: 'editor', + }, + }) + + const provider = await prisma.provider.create({ + data: { + name: 'google', + teamId: team.id, + config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, + }, + }) + + const model = await prisma.model.create({ + data: { + modelId: 'gemini', + name: 'Gemini', + providerId: provider.id, + config: { + input: ['text', 'image'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + reasoning: false, + }, + }, + }) + + await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'chat', + enabled: true, + providerId: provider.id, + modelId: model.id, + config: { + provider: 'google', + model: 'gemini', + }, + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'chat-asset.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + }, + }) + + // Create a regular user who authors the comment + const regularUser = await prisma.user.create({ + data: { + name: 'Regular User', + email: 'user@example.com', + type: 'human', + }, + }) + + // Create AgentSession first + await prisma.agentSession.create({ + data: { + id: 'chat-session-123', + agentId: agentUser.id, + userId: regularUser.id, + cwd: process.cwd(), + assetId: asset.id, + }, + }) + + // Create the User Comment + const userComment = await prisma.assetComment.create({ + data: { + assetId: asset.id, + message: 'Hello E2E Agent!', + creatorId: regularUser.id, + sessionId: 'chat-session-123', + }, + }) + + // 2. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'chat', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agent: { + agentId: agentUser.id, + sessionId: 'chat-session-123', + userCommentId: userComment.id, + userId: regularUser.id, + }, + }, + }, + }) + + // 3. Wait for workflow to complete + console.log( + `Submitted E2E Chat Workflow Task with Comment. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 4. Verification + expect(completedTask.status).toBe('completed') + + // Verify reply comment in DB + const comments = await prisma.assetComment.findMany({ + where: { + assetId: asset.id, + replyToId: userComment.id, + }, + }) + expect(comments.length).toBeGreaterThan(0) + expect(comments[0].message).toBe('E2E Agent Chat Response Success') + expect(comments[0].creatorId).toBe(agentUser.id) + }, 50000) + + it('should run agentChat workflow with direct prompt successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Direct Chat Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Direct Chat Project', teamId: team.id }, + }) + + const agentUser = await prisma.user.create({ + data: { + name: 'E2E Direct Chat Agent User', + email: 'e2e-dir-chat-agent@shumai.ai', + type: 'agent', + }, + }) + + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: agentUser.id, + role: 'editor', + }, + }) + + const provider = await prisma.provider.create({ + data: { + name: 'google', + teamId: team.id, + config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, + }, + }) + + const model = await prisma.model.create({ + data: { + modelId: 'gemini', + name: 'Gemini', + providerId: provider.id, + config: { + input: ['text'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + reasoning: false, + }, + }, + }) + + await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'chat', + enabled: true, + providerId: provider.id, + modelId: model.id, + config: { + provider: 'google', + model: 'gemini', + }, + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'direct-chat-asset.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + }, + }) + + const regularUser = await prisma.user.create({ + data: { + name: 'Regular User 2', + email: 'user2@example.com', + type: 'human', + }, + }) + + // Create AgentSession first + await prisma.agentSession.create({ + data: { + id: 'direct-session-456', + agentId: agentUser.id, + userId: regularUser.id, + cwd: process.cwd(), + assetId: asset.id, + }, + }) + + // 2. Create Workflow Task (without userCommentId, but with prompt/sessionId) + const task = await prisma.workflowTask.create({ + data: { + type: 'chat', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agent: { + agentId: agentUser.id, + sessionId: 'direct-session-456', + prompt: 'Direct test message', + userId: regularUser.id, + }, + }, + }, + }) + + // 3. Wait for workflow to complete + console.log(`Submitted E2E Direct Chat Workflow Task. ID: ${task.id}. Awaiting completion...`) + const completedTask = await workflowService.executeWait(task, 45000) + + // 4. Verification + expect(completedTask.status).toBe('completed') + expect((completedTask.output as unknown as { sessionId: string })?.sessionId).toBe( + 'direct-session-456', + ) + }, 50000) +}) diff --git a/packages/e2e/workflow/agent-embedding.test.ts b/packages/e2e/workflow/agent-embedding.test.ts new file mode 100644 index 00000000..9c82caba --- /dev/null +++ b/packages/e2e/workflow/agent-embedding.test.ts @@ -0,0 +1,481 @@ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { prisma, WorkflowTaskStatus } from '@shumai/db' +import { setupTestDbHooks } from '@shumai/db/test' +import { workflowService, TaskQueueAgent, TaskQueueTranscode } from '@shumai/workflow-core' +import { initAgentWorkflows } from '@shumai/agent' +import { initTranscodeWorkflows } from '@shumai/transcode' +import { s3Service } from '@shumai/core/src/s3/s3' +import { fileURLToPath } from 'url' +import * as path from 'path' +import * as fs from 'fs' + +const mockEmbedContent = vi.fn().mockResolvedValue({ + embeddings: [ + { + values: Array(1536).fill(0.2), + }, + ], +}) + +vi.mock('@google/genai', () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + GoogleGenAI: class { + models = { + embedContent: mockEmbedContent, + } + }, + } +}) + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) +const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts') +const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts') +const fixturesDir = path.resolve(currentDir, '../fixtures') + +describe.each(['local', 'temporal'] as const)( + 'Workflow E2E - agentEmbeddingMedia (executor: %s)', + (mode) => { + setupTestDbHooks() + + let agentWorkerPromise: Promise | null = null + let transcodeWorkerPromise: Promise | null = null + + beforeAll(async () => { + // Set bucket environment + process.env.S3_BUCKET = 'shumai-e2e-test-bucket-embedding' + process.env.GEMINI_API_KEY = 'dummy-key' + + // Configure the workflow service executor dynamically + workflowService.setExecutorType(mode) + + // Initialize registries + initAgentWorkflows() + initTranscodeWorkflows() + + if (mode === 'temporal') { + console.log('Starting background workers for Temporal E2E tests...') + agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, { + workflowsPath: agentWorkflowsPath, + }) + transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, { + workflowsPath: transcodeWorkflowsPath, + }) + + // Wait a moment for workers to establish connections + await new Promise((resolve) => setTimeout(resolve, 2000)) + } else { + console.log('Starting local workflow service polling...') + workflowService.start() + } + }) + + afterAll(async () => { + if (mode === 'temporal') { + console.log('Shutting down Temporal workers...') + await workflowService.shutdownWorkers() + await Promise.all([agentWorkerPromise, transcodeWorkerPromise].filter(Boolean)) + } + + workflowService.close() + + // Restore spied methods + vi.restoreAllMocks() + + // Clean up E2E files in local filesystem storage + try { + console.log('Cleaning up local E2E storage files...') + await s3Service.deletePrefix('shumai-e2e-test-bucket-embedding', '') + } catch (err) { + console.error('Failed to clean up E2E storage folder:', err) + } + }) + + it('should run agentEmbeddingMedia workflow for an image asset successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Image Embedding Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Image Embedding Project', teamId: team.id }, + }) + + const agentUser = await prisma.user.create({ + data: { + name: 'E2E Image Embedding Agent User', + email: 'e2e-img-emb-agent@shumai.ai', + type: 'agent', + }, + }) + + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: agentUser.id, + role: 'editor', + }, + }) + + const provider = await prisma.provider.create({ + data: { + name: 'google', + teamId: team.id, + config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, + }, + }) + + const model = await prisma.model.create({ + data: { + modelId: 'gemini', + name: 'Gemini', + providerId: provider.id, + config: { + input: ['text', 'image'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + reasoning: false, + }, + }, + }) + + await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'embedding', + enabled: true, + providerId: provider.id, + modelId: model.id, + config: { + provider: 'google', + model: 'gemini', + }, + }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/test-image.png', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'test-image.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const pngPath = path.join(fixturesDir, 'small.png') + const pngBuffer = fs.readFileSync(pngPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-embedding', + 'projects/e2e/test-image.png', + pngBuffer, + pngBuffer.length, + 'image/png', + ) + + // 3. Create Workflow Task (ai_embedding, pending) + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agent: { agentId: agentUser.id }, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Image Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + // Verify AssetComment indicates success + const comments = await prisma.assetComment.findMany({ + where: { assetId: asset.id }, + orderBy: { createdAt: 'asc' }, + }) + expect(comments.length).toBeGreaterThan(0) + const finalComment = comments[comments.length - 1] + expect(finalComment.message).toBe('Embedding completed successfully.') + + // Verify raw embedding is saved in DB + const dbEmbeddings = (await prisma.$queryRaw` + SELECT id, asset_id as "assetId", start_time as "startTime", end_time as "endTime" + FROM asset_embeddings + WHERE asset_id = ${asset.id} + `) as { id: string; startTime: number | null; endTime: number | null }[] + expect(dbEmbeddings.length).toBe(1) + expect(dbEmbeddings[0].startTime).toBeNull() + expect(dbEmbeddings[0].endTime).toBeNull() + }, 50000) + + it('should run agentEmbeddingMedia workflow for a video asset successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Video Embedding Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Video Embedding Project', teamId: team.id }, + }) + + const agentUser = await prisma.user.create({ + data: { + name: 'E2E Video Embedding Agent User', + email: 'e2e-vid-emb-agent@shumai.ai', + type: 'agent', + }, + }) + + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: agentUser.id, + role: 'editor', + }, + }) + + const provider = await prisma.provider.create({ + data: { + name: 'google', + teamId: team.id, + config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, + }, + }) + + const model = await prisma.model.create({ + data: { + modelId: 'gemini', + name: 'Gemini', + providerId: provider.id, + config: { + input: ['text', 'image'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + reasoning: false, + }, + }, + }) + + await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'embedding', + enabled: true, + providerId: provider.id, + modelId: model.id, + config: { + provider: 'google', + model: 'gemini', + }, + }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/test-video.mp4', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'test-video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + media: { duration: 1.0 } as unknown as PrismaJson.MediaInfo, // 1-second video + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const mp4Path = path.join(fixturesDir, 'small.mp4') + const mp4Buffer = fs.readFileSync(mp4Path) + await s3Service.putObject( + 'shumai-e2e-test-bucket-embedding', + 'projects/e2e/test-video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + + // 3. Create Workflow Task (ai_embedding, pending) + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agent: { agentId: agentUser.id }, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Video Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + // Verify raw embedding is saved in DB with start and end times + const dbEmbeddings = (await prisma.$queryRaw` + SELECT id, asset_id as "assetId", start_time as "startTime", end_time as "endTime" + FROM asset_embeddings + WHERE asset_id = ${asset.id} + ORDER BY start_time ASC + `) as { id: string; startTime: number | null; endTime: number | null }[] + expect(dbEmbeddings.length).toBeGreaterThan(0) + expect(dbEmbeddings[0].startTime).toBe(0) + expect(dbEmbeddings[0].endTime).toBe(1) + }, 50000) + + it('should fail agentEmbeddingMedia workflow for an unsupported audio asset', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Audio Embedding Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Audio Embedding Project', teamId: team.id }, + }) + + const agentUser = await prisma.user.create({ + data: { + name: 'E2E Audio Embedding Agent User', + email: 'e2e-aud-emb-agent@shumai.ai', + type: 'agent', + }, + }) + + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: agentUser.id, + role: 'editor', + }, + }) + + const provider = await prisma.provider.create({ + data: { + name: 'google', + teamId: team.id, + config: { api: 'google-generative-ai', apiKey: 'dummy-google-api-key' }, + }, + }) + + const model = await prisma.model.create({ + data: { + modelId: 'gemini', + name: 'Gemini', + providerId: provider.id, + config: { + input: ['text'], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + reasoning: false, + }, + }, + }) + + await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'embedding', + enabled: true, + providerId: provider.id, + modelId: model.id, + config: { + provider: 'google', + model: 'gemini', + }, + }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/test-audio.wav', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'test-audio.wav', + type: 'file', + status: 'uploaded', + mediaType: 'audio/wav', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const wavPath = path.join(fixturesDir, 'small.wav') + const wavBuffer = fs.readFileSync(wavPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-embedding', + 'projects/e2e/test-audio.wav', + wavBuffer, + wavBuffer.length, + 'audio/wav', + ) + + // 3. Create Workflow Task (ai_embedding, pending) + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agent: { agentId: agentUser.id }, + }, + }, + }) + + // 4. Wait for workflow to fail and verify error message + console.log( + `Submitted E2E Audio Embedding Workflow Task. ID: ${task.id}. Awaiting failure...`, + ) + await expect(workflowService.executeWait(task, 45000)).rejects.toThrow( + 'unsupported media type for embeddings: audio/wav', + ) + + // Verify task status in DB is failed + const dbTask = await prisma.workflowTask.findUnique({ + where: { id: task.id }, + }) + expect(dbTask?.status).toBe(WorkflowTaskStatus.failed) + }, 50000) + }, +) diff --git a/packages/e2e/workflow/agent-tool-call.test.ts b/packages/e2e/workflow/agent-tool-call.test.ts new file mode 100644 index 00000000..5fcaffe3 --- /dev/null +++ b/packages/e2e/workflow/agent-tool-call.test.ts @@ -0,0 +1,146 @@ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { prisma } from '@shumai/db' +import { setupTestDbHooks } from '@shumai/db/test' +import { workflowService, TaskQueueAgent } from '@shumai/workflow-core' +import { initAgentWorkflows } from '@shumai/agent' +import { s3Service } from '@shumai/core/src/s3/s3' +import { fileURLToPath } from 'url' +import * as path from 'path' + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) +const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts') + +describe.each(['local', 'temporal'] as const)( + 'Workflow E2E - agentToolCall (executor: %s)', + (mode) => { + setupTestDbHooks() + + let agentWorkerPromise: Promise | null = null + + beforeAll(async () => { + process.env.S3_BUCKET = 'shumai-e2e-test-bucket-toolcall' + process.env.GEMINI_API_KEY = 'dummy-key' + + workflowService.setExecutorType(mode) + initAgentWorkflows() + + if (mode === 'temporal') { + console.log('Starting background worker for tool call Temporal E2E tests...') + agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, { + workflowsPath: agentWorkflowsPath, + }) + await new Promise((resolve) => setTimeout(resolve, 2000)) + } else { + console.log('Starting local workflow service polling...') + workflowService.start() + } + }) + + afterAll(async () => { + if (mode === 'temporal') { + console.log('Shutting down Temporal workers...') + await workflowService.shutdownWorkers() + await Promise.all([agentWorkerPromise].filter(Boolean)) + } + workflowService.close() + vi.restoreAllMocks() + try { + console.log('Cleaning up local E2E storage files...') + await s3Service.deletePrefix('shumai-e2e-test-bucket-toolcall', '') + } catch (err) { + console.error('Failed to clean up E2E storage folder:', err) + } + }) + + it('should run agentToolCall workflow and execute list_assets successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Tool Call Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Tool Call Project', teamId: team.id }, + }) + + const regularUser = await prisma.user.create({ + data: { + name: 'Regular Editor User', + email: 'editor@example.com', + type: 'human', + }, + }) + + // Setup team member with owner/editor role and team scope + await prisma.teamMember.create({ + data: { + teamId: team.id, + userId: regularUser.id, + role: 'editor', + scope: 'team', + }, + }) + + // Create a folder asset (parent) + const parentFolder = await prisma.asset.create({ + data: { + name: 'TestParentFolder', + type: 'folder', + status: 'processed', + projectId: project.id, + }, + }) + + // Create a child file asset inside the parent folder + const childFile = await prisma.asset.create({ + data: { + name: 'TestChildFile.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + parentId: parentFolder.id, + }, + }) + + // 2. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'agent_tool_call', + status: 'pending', + assetId: parentFolder.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + agentToolCall: { + toolName: 'list_assets', + args: { + parent: parentFolder.id, + page: 1, + pageSize: 20, + type: 'all', + }, + userId: regularUser.id, + }, + }, + }, + }) + + // 3. Wait for workflow to complete + console.log(`Submitted E2E Tool Call Workflow Task. ID: ${task.id}. Awaiting completion...`) + const completedTask = await workflowService.executeWait(task, 45000) + + // 4. Verification + expect(completedTask.status).toBe('completed') + + // Verify task output contains the list of files, and childFile.id is present in assets + const output = completedTask.output as unknown as { assets: { id: string }[] } + expect(output).toBeDefined() + expect(output.assets).toBeDefined() + expect(output.assets.length).toBeGreaterThan(0) + + const assetIds = output.assets.map((a) => a.id) + expect(assetIds).toContain(childFile.id) + }, 50000) + }, +) diff --git a/packages/e2e/workflow/query-embedding-for-search.test.ts b/packages/e2e/workflow/query-embedding-for-search.test.ts new file mode 100644 index 00000000..897ef624 --- /dev/null +++ b/packages/e2e/workflow/query-embedding-for-search.test.ts @@ -0,0 +1,127 @@ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { prisma } from '@shumai/db' +import { setupTestDbHooks } from '@shumai/db/test' +import { workflowService, TaskQueueAgent } from '@shumai/workflow-core' +import { initAgentWorkflows } from '@shumai/agent' +import { s3Service } from '@shumai/core/src/s3/s3' +const mockEmbedContent = vi.fn().mockResolvedValue({ + embeddings: [ + { + values: Array(1536).fill(0.35), + }, + ], +}) + +vi.mock('@google/genai', () => { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + GoogleGenAI: class { + models = { + embedContent: mockEmbedContent, + } + }, + } +}) + +import { fileURLToPath } from 'url' +import * as path from 'path' + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) +const agentWorkflowsPath = path.resolve(currentDir, '../../../apps/agent/src/workflows.ts') + +describe.each(['local', 'temporal'] as const)( + 'Workflow E2E - queryEmbeddingForSearch (executor: %s)', + (mode) => { + setupTestDbHooks() + + let agentWorkerPromise: Promise | null = null + + beforeAll(async () => { + process.env.S3_BUCKET = 'shumai-e2e-test-bucket-query' + process.env.GEMINI_API_KEY = 'dummy-key' + + workflowService.setExecutorType(mode) + initAgentWorkflows() + + if (mode === 'temporal') { + console.log('Starting background worker for query embedding Temporal E2E tests...') + agentWorkerPromise = workflowService.startWorkers(TaskQueueAgent, { + workflowsPath: agentWorkflowsPath, + }) + await new Promise((resolve) => setTimeout(resolve, 2000)) + } else { + console.log('Starting local workflow service polling...') + workflowService.start() + } + }) + + afterAll(async () => { + if (mode === 'temporal') { + console.log('Shutting down Temporal workers...') + await workflowService.shutdownWorkers() + await Promise.all([agentWorkerPromise].filter(Boolean)) + } + workflowService.close() + vi.restoreAllMocks() + try { + console.log('Cleaning up local E2E storage files...') + await s3Service.deletePrefix('shumai-e2e-test-bucket-query', '') + } catch (err) { + console.error('Failed to clean up E2E storage folder:', err) + } + }) + + it('should run queryEmbeddingForSearch workflow and return embedding successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Query Embedding Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Query Embedding Project', teamId: team.id }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'dummy-search-asset', + type: 'file', + status: 'uploaded', + mediaType: 'text/plain', + projectId: project.id, + }, + }) + + // 2. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'query_embedding_for_search', + status: 'pending', + teamId: team.id, + projectId: project.id, + assetId: asset.id, + payload: { + projectId: project.id, + queryEmbeddingForSearch: { + text: 'test search query', + }, + }, + }, + }) + + // 3. Wait for workflow to complete + console.log( + `Submitted E2E Query Embedding Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 4. Verification + expect(completedTask.status).toBe('completed') + + // Verify output payload contains correct embedding values and usage metadata + const output = completedTask.output as unknown as { embedding: number[] } + expect(output).toBeDefined() + expect(output.embedding).toEqual(Array(1536).fill(0.35)) + expect(completedTask.model).toBe('gemini-embedding-2') + }, 50000) + }, +) diff --git a/packages/e2e/workflow/transcode.test.ts b/packages/e2e/workflow/transcode.test.ts new file mode 100644 index 00000000..096fd73f --- /dev/null +++ b/packages/e2e/workflow/transcode.test.ts @@ -0,0 +1,459 @@ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { prisma, AssetStatus } from '@shumai/db' +import { setupTestDbHooks } from '@shumai/db/test' +import { workflowService, TaskQueueTranscode } from '@shumai/workflow-core' +import { initTranscodeWorkflows } from '@shumai/transcode' +import { s3Service } from '@shumai/core/src/s3/s3' +import { fileURLToPath } from 'url' +import * as path from 'path' +import * as fs from 'fs' + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) +const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts') +const fixturesDir = path.resolve(currentDir, '../fixtures') + +describe.each(['local', 'temporal'] as const)( + 'Workflow E2E - transcodeMedia (executor: %s)', + (mode) => { + setupTestDbHooks() + + let transcodeWorkerPromise: Promise | null = null + + beforeAll(async () => { + process.env.S3_BUCKET = 'shumai-e2e-test-bucket-transcode' + + workflowService.setExecutorType(mode) + initTranscodeWorkflows() + + if (mode === 'temporal') { + console.log('Starting background worker for transcode Temporal E2E tests...') + transcodeWorkerPromise = workflowService.startWorkers(TaskQueueTranscode, { + workflowsPath: transcodeWorkflowsPath, + }) + await new Promise((resolve) => setTimeout(resolve, 2000)) + } else { + console.log('Starting local workflow service polling...') + workflowService.start() + } + }) + + afterAll(async () => { + if (mode === 'temporal') { + console.log('Shutting down Temporal workers...') + await workflowService.shutdownWorkers() + await Promise.all([transcodeWorkerPromise].filter(Boolean)) + } + workflowService.close() + vi.restoreAllMocks() + try { + console.log('Cleaning up local E2E storage files...') + await s3Service.deletePrefix('shumai-e2e-test-bucket-transcode', '') + } catch (err) { + console.error('Failed to clean up E2E storage folder:', err) + } + }) + + it('should run transcodeMedia workflow for a video asset successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Video Transcode Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Video Transcode Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/video-trans.mp4', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'video-trans.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const mp4Path = path.join(fixturesDir, 'small.mp4') + const mp4Buffer = fs.readFileSync(mp4Path) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/video-trans.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + + // 3. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + thumbnail: false, + poster: true, + sprite: true, + }, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Video Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + // Verify Asset status is updated to processed + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + // Verify media info contains videoTranscodes, poster, sprite, and duration details + const mediaInfo = updatedAsset?.media as unknown as { + mimeType: string + duration: number + videoTranscodes: { key: string }[] + poster: unknown + sprite: unknown + } + expect(mediaInfo).toBeDefined() + expect(mediaInfo.mimeType).toBe('video/mp4') + expect(mediaInfo.duration).toBeCloseTo(1.0, 1) + expect(mediaInfo.videoTranscodes).toBeDefined() + expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0) + expect(mediaInfo.poster).toBeDefined() + expect(mediaInfo.sprite).toBeDefined() + }, 50000) + + it('should run transcodeMedia workflow for an image asset successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Image Transcode Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Image Transcode Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/image-trans.png', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'image-trans.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const pngPath = path.join(fixturesDir, 'small.png') + const pngBuffer = fs.readFileSync(pngPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/image-trans.png', + pngBuffer, + pngBuffer.length, + 'image/png', + ) + + // 3. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + thumbnail: true, + }, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Image Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + mimeType: string + imageTranscodes: unknown[] + thumbnail: unknown + } + expect(mediaInfo).toBeDefined() + expect(mediaInfo.mimeType).toBe('image/png') + expect(mediaInfo.imageTranscodes).toBeDefined() + expect(mediaInfo.imageTranscodes.length).toBeGreaterThan(0) + expect(mediaInfo.thumbnail).toBeDefined() + }, 50000) + + it('should run transcodeMedia workflow for an audio asset successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Audio Transcode Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Audio Transcode Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/audio-trans.wav', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'audio-trans.wav', + type: 'file', + status: 'uploaded', + mediaType: 'audio/wav', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const wavPath = path.join(fixturesDir, 'small.wav') + const wavBuffer = fs.readFileSync(wavPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/audio-trans.wav', + wavBuffer, + wavBuffer.length, + 'audio/wav', + ) + + // 3. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: {}, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Audio Transcode Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + mimeType: string + videoTranscodes: { key: string }[] + } + expect(mediaInfo).toBeDefined() + expect(mediaInfo.mimeType).toBe('audio/wav') + expect(mediaInfo.videoTranscodes).toBeDefined() + expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0) + // Audio proxy key should end with -audio-proxy.mp4 + expect(mediaInfo.videoTranscodes[0].key).toContain('-audio-proxy.mp4') + }, 50000) + + it('should run transcodeMedia screenshot extraction successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Video Screenshot Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Video Screenshot Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/video-shot.mp4', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'video-shot.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const mp4Path = path.join(fixturesDir, 'small.mp4') + const mp4Buffer = fs.readFileSync(mp4Path) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/video-shot.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + + // 3. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + screenshot: { + start: 0, + end: 1, + count: 1, + }, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Video Screenshot Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + // Verify task output contains list of screenshots + const output = completedTask.output as unknown as { screenshots: { key: string }[] } + expect(output).toBeDefined() + expect(output.screenshots).toBeDefined() + expect(output.screenshots.length).toBe(1) + expect(output.screenshots[0].key).toContain('screenshots/') + }, 50000) + + it('should run transcodeMedia image annotation overlay successfully', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Image Annotation Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Image Annotation Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/image-ann.png', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'image-ann.png', + type: 'file', + status: 'uploaded', + mediaType: 'image/png', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const pngPath = path.join(fixturesDir, 'small.png') + const pngBuffer = fs.readFileSync(pngPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/image-ann.png', + pngBuffer, + pngBuffer.length, + 'image/png', + ) + + // 3. Create Workflow Task + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + imageAnnotation: { + annotations: [ + { + type: 'box', + color: '#ff0000', + points: [ + [0.1, 0.1], + [0.9, 0.9], + ], + }, + ], + }, + }, + }, + }) + + // 4. Wait for workflow to complete + console.log( + `Submitted E2E Image Annotation Workflow Task. ID: ${task.id}. Awaiting completion...`, + ) + const completedTask = await workflowService.executeWait(task, 45000) + + // 5. Verification + expect(completedTask.status).toBe('completed') + + // Verify task output contains overlay image key + const output = completedTask.output as unknown as { key: string } + expect(output).toBeDefined() + expect(output.key).toContain('annotations/') + }, 50000) + }, +) diff --git a/packages/workflow-core/src/temporal-executor.ts b/packages/workflow-core/src/temporal-executor.ts index 1e2144ef..2ed8b621 100644 --- a/packages/workflow-core/src/temporal-executor.ts +++ b/packages/workflow-core/src/temporal-executor.ts @@ -33,6 +33,10 @@ export class TemporalExecutor implements Executor { workflowName = 'agentChatWorkflow' taskQueue = TaskQueueAgent break + case WorkflowTaskType.agent_tool_call: + workflowName = 'agentToolCallWorkflow' + taskQueue = TaskQueueAgent + break case WorkflowTaskType.transcode: workflowName = 'transcodeWorkflow' taskQueue = TaskQueueTranscode