feat(transcode): trigger ai metadata autofill post transcode

This commit is contained in:
yiling
2026-07-28 20:19:10 +08:00
parent d0d29b34ed
commit e51d86e8f0
11 changed files with 185 additions and 31 deletions
+4 -1
View File
@@ -8,6 +8,7 @@ import * as path from 'path'
import { promisify } from 'util'
import { ApplicationFailure } from '@temporalio/activity'
import { prisma } from '@shumai/db'
import { getProxyType } from '@shumai/core/src/utils/mime'
const execFileAsync = promisify(execFile)
@@ -88,7 +89,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
outputTokens: 0,
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
@@ -1,11 +1,12 @@
import { ApplicationFailure } from '@temporalio/workflow'
import type { WorkflowTask } from '@shumai/db'
import {
getActivities,
executeActivity,
getActivities,
TaskQueueAgent,
TaskQueueTranscode,
} from '@shumai/workflow-core'
import { ApplicationFailure } from '@temporalio/workflow'
import { getProxyType } from '@shumai/core/src/utils/mime'
export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
const {
@@ -64,7 +65,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
const projectId = asset.project.id
// 2. Prepare Data (Images)
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
-27
View File
@@ -6,8 +6,6 @@ import {
AssetType,
Prisma,
TaskStatus,
WorkflowTaskStatus,
WorkflowTaskType,
prisma,
} from '@shumai/db'
import {
@@ -292,15 +290,6 @@ export class UploadService {
})
if (!team) throw new Error('Team not found')
// Ai Tasks
const autofillAgent = await tx.agent.findFirst({
where: {
type: 'autofill',
enabled: true,
user: { teamMembers: { some: { teamId: team.id } } },
},
})
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
@@ -310,22 +299,6 @@ export class UploadService {
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
if (autofillAgent && (isVideo || isImage)) {
await tx.workflowTask.create({
data: {
assetId: asset.id,
type: WorkflowTaskType.ai_metadata_autofill,
status: WorkflowTaskStatus.pending,
teamId: team.id,
projectId,
payload: {
projectId,
agent: { agentId: autofillAgent.id },
},
},
})
}
if (!proxyType) {
await tx.asset.update({
where: { id: asset.id },
@@ -23,6 +23,7 @@ import {
generatePdfProxyActivity,
transcodeVideoChunkActivity,
deleteS3ObjectActivity,
createAutofillTaskIfEnabledActivity,
} from './transcode'
vi.mock('@shumai/core/src/s3/s3', () => ({
@@ -637,5 +638,55 @@ describe('Transcode Activities', () => {
key: 'files/proj-123/a-audio-proxy.mp4',
})
})
it('should create ai_metadata_autofill task when autofill agent exists and asset is image', async () => {
const user = await prisma.user.create({
data: { name: 'Test User Autofill', email: 'test-autofill-agent@test.com' },
})
const team = await prisma.team.create({
data: { name: 'Test Team Autofill' },
})
await prisma.teamMember.create({
data: { teamId: team.id, userId: user.id, role: 'owner' },
})
const project = await prisma.project.create({
data: { name: 'Test Project Autofill', teamId: team.id },
})
const autofillAgent = await prisma.agent.create({
data: {
id: user.id,
type: 'autofill',
enabled: true,
teamId: team.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
config: { provider: 'test', model: 'test' } as any,
},
})
const asset = await prisma.asset.create({
data: {
name: 'photo.png',
mediaType: 'image/png',
type: 'file',
status: 'processed',
projectId: project.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
media: { proxyType: 'image' } as any,
},
})
await createAutofillTaskIfEnabledActivity({
assetId: asset.id,
teamId: team.id,
projectId: project.id,
})
const task = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: 'ai_metadata_autofill' },
})
expect(task).toBeDefined()
expect(task?.status).toBe('pending')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
})
})
})
@@ -769,6 +769,82 @@ export async function createEmbeddingTaskIfEnabledActivity(
})
}
export interface CreateAutofillTaskIfEnabledParams {
assetId: string
teamId: string | null
projectId: string | null
}
export async function createAutofillTaskIfEnabledActivity(
params: CreateAutofillTaskIfEnabledParams,
): Promise<void> {
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: {
project: true,
},
})
if (!asset) {
return
}
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
if (!isVideo && !isImage) {
return
}
const teamId = params.teamId || asset.project?.teamId || null
const projectId = params.projectId || asset.projectId
if (!teamId) {
return
}
// Check if there is an active autofill agent for the team
const autofillAgent = await prisma.agent.findFirst({
where: {
type: 'autofill',
enabled: true,
user: { teamMembers: { some: { teamId } } },
},
})
if (!autofillAgent) {
return
}
// Check if a pending/processing autofill task already exists for this asset to avoid duplicates
const existing = await prisma.workflowTask.findFirst({
where: {
assetId: params.assetId,
type: WorkflowTaskType.ai_metadata_autofill,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
})
if (existing) {
return
}
// Create autofill task
await prisma.workflowTask.create({
data: {
assetId: params.assetId,
type: WorkflowTaskType.ai_metadata_autofill,
status: WorkflowTaskStatus.pending,
teamId,
projectId,
payload: {
projectId: projectId ?? '',
agent: { agentId: autofillAgent.id },
},
},
})
}
export interface TranscodeVideoChunkParams {
assetId: string
filePath: string
@@ -38,6 +38,9 @@ describe('transcodeImageWorkflow', () => {
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
}),
createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createAutofillTaskIfEnabledActivity',
}),
}
beforeEach(() => {
@@ -145,5 +148,11 @@ describe('transcodeImageWorkflow', () => {
teamId: 'team-1',
projectId: 'proj-1',
})
expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-image',
teamId: 'team-1',
projectId: 'proj-1',
})
})
})
@@ -23,6 +23,7 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void>
updateAssetMediaActivity,
downloadMediaToTmpActivity,
createEmbeddingTaskIfEnabledActivity,
createAutofillTaskIfEnabledActivity,
} = getActivities()
await executeActivity(workerQueue, updateAssetStatusActivity, {
@@ -96,6 +97,12 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void>
projectId: task.projectId,
})
await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
await completeTask(workerQueue, task.id)
} catch (err) {
console.error(`transcodeImageWorkflow failed for task ${task.id}:`, err)
@@ -41,6 +41,9 @@ describe('transcodePdfWorkflow', () => {
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
}),
createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createAutofillTaskIfEnabledActivity',
}),
}
beforeEach(() => {
@@ -150,5 +153,11 @@ describe('transcodePdfWorkflow', () => {
assetId: 'asset-pdf',
status: AssetStatus.processed,
})
expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
teamId: 'team-1',
projectId: 'proj-1',
})
})
})
@@ -24,6 +24,7 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
downloadMediaToTmpActivity,
generatePdfProxyActivity,
createEmbeddingTaskIfEnabledActivity,
createAutofillTaskIfEnabledActivity,
} = getActivities()
await executeActivity(workerQueue, updateAssetStatusActivity, {
@@ -108,6 +109,12 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
projectId: task.projectId,
})
await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
await completeTask(workerQueue, task.id)
} catch (err) {
console.error(`transcodePdfWorkflow failed for task ${task.id}:`, err)
@@ -41,6 +41,9 @@ describe('transcodeVideoWorkflow', () => {
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createEmbeddingTaskIfEnabledActivity',
}),
createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), {
_activityName: 'createAutofillTaskIfEnabledActivity',
}),
}
beforeEach(() => {
@@ -151,6 +154,12 @@ describe('transcodeVideoWorkflow', () => {
projectId: 'proj-1',
})
expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({
assetId: 'asset-1',
teamId: 'team-1',
projectId: 'proj-1',
})
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
taskId: 'task-1',
status: WorkflowTaskStatus.completed,
@@ -27,6 +27,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
updateAssetMediaActivity,
downloadMediaToTmpActivity,
createEmbeddingTaskIfEnabledActivity,
createAutofillTaskIfEnabledActivity,
} = getActivities()
await executeActivity(workerQueue, updateAssetStatusActivity, {
@@ -161,6 +162,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
projectId: task.projectId,
})
await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, {
assetId: asset.id,
teamId: task.teamId,
projectId: task.projectId,
})
await completeTask(workerQueue, task.id)
} catch (err) {
console.error(`transcodeVideoWorkflow failed for task ${task.id}:`, err)