diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index 27728cd9..3d245544 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -8,6 +8,7 @@ import * as path from 'path' import { promisify } from 'util' import { ApplicationFailure } from '@temporalio/activity' import { prisma } from '@shumai/db' +import { getProxyType } from '@shumai/core/src/utils/mime' const execFileAsync = promisify(execFile) @@ -88,7 +89,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams) outputTokens: 0, } - const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType + const proxyType = + (asset.media as PrismaJson.MediaInfo | null)?.proxyType || + getProxyType(asset.mediaType, asset.name) const isImage = proxyType === 'image' const isVideo = proxyType === 'video' diff --git a/packages/agent/src/workflows/agent-autofill.ts b/packages/agent/src/workflows/agent-autofill.ts index c2d48186..27c58277 100644 --- a/packages/agent/src/workflows/agent-autofill.ts +++ b/packages/agent/src/workflows/agent-autofill.ts @@ -1,11 +1,12 @@ -import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' import { - getActivities, executeActivity, + getActivities, TaskQueueAgent, TaskQueueTranscode, } from '@shumai/workflow-core' +import { ApplicationFailure } from '@temporalio/workflow' +import { getProxyType } from '@shumai/core/src/utils/mime' export async function agentAutofillMedia(task: WorkflowTask): Promise { const { @@ -64,7 +65,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { const projectId = asset.project.id // 2. Prepare Data (Images) - const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType + const proxyType = + (asset.media as PrismaJson.MediaInfo | null)?.proxyType || + getProxyType(asset.mediaType, asset.name) const isImage = proxyType === 'image' const isVideo = proxyType === 'video' diff --git a/packages/core/src/upload/upload.ts b/packages/core/src/upload/upload.ts index e1be4dfd..a7595c9c 100644 --- a/packages/core/src/upload/upload.ts +++ b/packages/core/src/upload/upload.ts @@ -6,8 +6,6 @@ import { AssetType, Prisma, TaskStatus, - WorkflowTaskStatus, - WorkflowTaskType, prisma, } from '@shumai/db' import { @@ -292,15 +290,6 @@ export class UploadService { }) if (!team) throw new Error('Team not found') - // Ai Tasks - const autofillAgent = await tx.agent.findFirst({ - where: { - type: 'autofill', - enabled: true, - user: { teamMembers: { some: { teamId: team.id } } }, - }, - }) - const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType || getProxyType(asset.mediaType, asset.name) @@ -310,22 +299,6 @@ export class UploadService { const isAudio = proxyType === 'audio' const isPdf = proxyType === 'pdf' - if (autofillAgent && (isVideo || isImage)) { - await tx.workflowTask.create({ - data: { - assetId: asset.id, - type: WorkflowTaskType.ai_metadata_autofill, - status: WorkflowTaskStatus.pending, - teamId: team.id, - projectId, - payload: { - projectId, - agent: { agentId: autofillAgent.id }, - }, - }, - }) - } - if (!proxyType) { await tx.asset.update({ where: { id: asset.id }, diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 1e9cc15e..905f7bb5 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -23,6 +23,7 @@ import { generatePdfProxyActivity, transcodeVideoChunkActivity, deleteS3ObjectActivity, + createAutofillTaskIfEnabledActivity, } from './transcode' vi.mock('@shumai/core/src/s3/s3', () => ({ @@ -637,5 +638,55 @@ describe('Transcode Activities', () => { key: 'files/proj-123/a-audio-proxy.mp4', }) }) + + it('should create ai_metadata_autofill task when autofill agent exists and asset is image', async () => { + const user = await prisma.user.create({ + data: { name: 'Test User Autofill', email: 'test-autofill-agent@test.com' }, + }) + const team = await prisma.team.create({ + data: { name: 'Test Team Autofill' }, + }) + await prisma.teamMember.create({ + data: { teamId: team.id, userId: user.id, role: 'owner' }, + }) + const project = await prisma.project.create({ + data: { name: 'Test Project Autofill', teamId: team.id }, + }) + const autofillAgent = await prisma.agent.create({ + data: { + id: user.id, + type: 'autofill', + enabled: true, + teamId: team.id, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + config: { provider: 'test', model: 'test' } as any, + }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'photo.png', + mediaType: 'image/png', + type: 'file', + status: 'processed', + projectId: project.id, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + media: { proxyType: 'image' } as any, + }, + }) + + await createAutofillTaskIfEnabledActivity({ + assetId: asset.id, + teamId: team.id, + projectId: project.id, + }) + + const task = await prisma.workflowTask.findFirst({ + where: { assetId: asset.id, type: 'ai_metadata_autofill' }, + }) + expect(task).toBeDefined() + expect(task?.status).toBe('pending') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id) + }) }) }) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index bb5f7932..21af8934 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -769,6 +769,82 @@ export async function createEmbeddingTaskIfEnabledActivity( }) } +export interface CreateAutofillTaskIfEnabledParams { + assetId: string + teamId: string | null + projectId: string | null +} + +export async function createAutofillTaskIfEnabledActivity( + params: CreateAutofillTaskIfEnabledParams, +): Promise { + const asset = await prisma.asset.findUnique({ + where: { id: params.assetId }, + include: { + project: true, + }, + }) + if (!asset) { + return + } + + const proxyType = + (asset.media as PrismaJson.MediaInfo | null)?.proxyType || + getProxyType(asset.mediaType, asset.name) + const isVideo = proxyType === 'video' + const isImage = proxyType === 'image' + if (!isVideo && !isImage) { + return + } + + const teamId = params.teamId || asset.project?.teamId || null + const projectId = params.projectId || asset.projectId + + if (!teamId) { + return + } + + // Check if there is an active autofill agent for the team + const autofillAgent = await prisma.agent.findFirst({ + where: { + type: 'autofill', + enabled: true, + user: { teamMembers: { some: { teamId } } }, + }, + }) + + if (!autofillAgent) { + return + } + + // Check if a pending/processing autofill task already exists for this asset to avoid duplicates + const existing = await prisma.workflowTask.findFirst({ + where: { + assetId: params.assetId, + type: WorkflowTaskType.ai_metadata_autofill, + status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] }, + }, + }) + if (existing) { + return + } + + // Create autofill task + await prisma.workflowTask.create({ + data: { + assetId: params.assetId, + type: WorkflowTaskType.ai_metadata_autofill, + status: WorkflowTaskStatus.pending, + teamId, + projectId, + payload: { + projectId: projectId ?? '', + agent: { agentId: autofillAgent.id }, + }, + }, + }) +} + export interface TranscodeVideoChunkParams { assetId: string filePath: string diff --git a/packages/transcode/src/workflows/transcode-image.test.ts b/packages/transcode/src/workflows/transcode-image.test.ts index 84740ba3..8c3b199f 100644 --- a/packages/transcode/src/workflows/transcode-image.test.ts +++ b/packages/transcode/src/workflows/transcode-image.test.ts @@ -38,6 +38,9 @@ describe('transcodeImageWorkflow', () => { createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), { _activityName: 'createEmbeddingTaskIfEnabledActivity', }), + createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), { + _activityName: 'createAutofillTaskIfEnabledActivity', + }), } beforeEach(() => { @@ -145,5 +148,11 @@ describe('transcodeImageWorkflow', () => { teamId: 'team-1', projectId: 'proj-1', }) + + expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({ + assetId: 'asset-image', + teamId: 'team-1', + projectId: 'proj-1', + }) }) }) diff --git a/packages/transcode/src/workflows/transcode-image.ts b/packages/transcode/src/workflows/transcode-image.ts index 1d3dd6ff..ab662335 100644 --- a/packages/transcode/src/workflows/transcode-image.ts +++ b/packages/transcode/src/workflows/transcode-image.ts @@ -23,6 +23,7 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise updateAssetMediaActivity, downloadMediaToTmpActivity, createEmbeddingTaskIfEnabledActivity, + createAutofillTaskIfEnabledActivity, } = getActivities() await executeActivity(workerQueue, updateAssetStatusActivity, { @@ -96,6 +97,12 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise projectId: task.projectId, }) + await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, { + assetId: asset.id, + teamId: task.teamId, + projectId: task.projectId, + }) + await completeTask(workerQueue, task.id) } catch (err) { console.error(`transcodeImageWorkflow failed for task ${task.id}:`, err) diff --git a/packages/transcode/src/workflows/transcode-pdf.test.ts b/packages/transcode/src/workflows/transcode-pdf.test.ts index d56ad0a0..76091bce 100644 --- a/packages/transcode/src/workflows/transcode-pdf.test.ts +++ b/packages/transcode/src/workflows/transcode-pdf.test.ts @@ -41,6 +41,9 @@ describe('transcodePdfWorkflow', () => { createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), { _activityName: 'createEmbeddingTaskIfEnabledActivity', }), + createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), { + _activityName: 'createAutofillTaskIfEnabledActivity', + }), } beforeEach(() => { @@ -150,5 +153,11 @@ describe('transcodePdfWorkflow', () => { assetId: 'asset-pdf', status: AssetStatus.processed, }) + + expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({ + assetId: 'asset-pdf', + teamId: 'team-1', + projectId: 'proj-1', + }) }) }) diff --git a/packages/transcode/src/workflows/transcode-pdf.ts b/packages/transcode/src/workflows/transcode-pdf.ts index 071fe2b1..eaff5723 100644 --- a/packages/transcode/src/workflows/transcode-pdf.ts +++ b/packages/transcode/src/workflows/transcode-pdf.ts @@ -24,6 +24,7 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise { downloadMediaToTmpActivity, generatePdfProxyActivity, createEmbeddingTaskIfEnabledActivity, + createAutofillTaskIfEnabledActivity, } = getActivities() await executeActivity(workerQueue, updateAssetStatusActivity, { @@ -108,6 +109,12 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise { projectId: task.projectId, }) + await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, { + assetId: asset.id, + teamId: task.teamId, + projectId: task.projectId, + }) + await completeTask(workerQueue, task.id) } catch (err) { console.error(`transcodePdfWorkflow failed for task ${task.id}:`, err) diff --git a/packages/transcode/src/workflows/transcode-video.test.ts b/packages/transcode/src/workflows/transcode-video.test.ts index a70f46d9..2f7bb26a 100644 --- a/packages/transcode/src/workflows/transcode-video.test.ts +++ b/packages/transcode/src/workflows/transcode-video.test.ts @@ -41,6 +41,9 @@ describe('transcodeVideoWorkflow', () => { createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), { _activityName: 'createEmbeddingTaskIfEnabledActivity', }), + createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), { + _activityName: 'createAutofillTaskIfEnabledActivity', + }), } beforeEach(() => { @@ -151,6 +154,12 @@ describe('transcodeVideoWorkflow', () => { projectId: 'proj-1', }) + expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({ + assetId: 'asset-1', + teamId: 'team-1', + projectId: 'proj-1', + }) + expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({ taskId: 'task-1', status: WorkflowTaskStatus.completed, diff --git a/packages/transcode/src/workflows/transcode-video.ts b/packages/transcode/src/workflows/transcode-video.ts index 5c47c7a1..afbf56c3 100644 --- a/packages/transcode/src/workflows/transcode-video.ts +++ b/packages/transcode/src/workflows/transcode-video.ts @@ -27,6 +27,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise updateAssetMediaActivity, downloadMediaToTmpActivity, createEmbeddingTaskIfEnabledActivity, + createAutofillTaskIfEnabledActivity, } = getActivities() await executeActivity(workerQueue, updateAssetStatusActivity, { @@ -161,6 +162,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise projectId: task.projectId, }) + await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, { + assetId: asset.id, + teamId: task.teamId, + projectId: task.projectId, + }) + await completeTask(workerQueue, task.id) } catch (err) { console.error(`transcodeVideoWorkflow failed for task ${task.id}:`, err)