mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 03:49:33 +00:00
feat(transcode): trigger ai metadata autofill post transcode
This commit is contained in:
@@ -8,6 +8,7 @@ import * as path from 'path'
|
||||
import { promisify } from 'util'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
@@ -88,7 +89,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import type { WorkflowTask } from '@shumai/db'
|
||||
import {
|
||||
getActivities,
|
||||
executeActivity,
|
||||
getActivities,
|
||||
TaskQueueAgent,
|
||||
TaskQueueTranscode,
|
||||
} from '@shumai/workflow-core'
|
||||
import { ApplicationFailure } from '@temporalio/workflow'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
|
||||
export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
const {
|
||||
@@ -64,7 +65,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
const projectId = asset.project.id
|
||||
|
||||
// 2. Prepare Data (Images)
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
|
||||
@@ -6,8 +6,6 @@ import {
|
||||
AssetType,
|
||||
Prisma,
|
||||
TaskStatus,
|
||||
WorkflowTaskStatus,
|
||||
WorkflowTaskType,
|
||||
prisma,
|
||||
} from '@shumai/db'
|
||||
import {
|
||||
@@ -292,15 +290,6 @@ export class UploadService {
|
||||
})
|
||||
if (!team) throw new Error('Team not found')
|
||||
|
||||
// Ai Tasks
|
||||
const autofillAgent = await tx.agent.findFirst({
|
||||
where: {
|
||||
type: 'autofill',
|
||||
enabled: true,
|
||||
user: { teamMembers: { some: { teamId: team.id } } },
|
||||
},
|
||||
})
|
||||
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
@@ -310,22 +299,6 @@ export class UploadService {
|
||||
const isAudio = proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
|
||||
if (autofillAgent && (isVideo || isImage)) {
|
||||
await tx.workflowTask.create({
|
||||
data: {
|
||||
assetId: asset.id,
|
||||
type: WorkflowTaskType.ai_metadata_autofill,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
teamId: team.id,
|
||||
projectId,
|
||||
payload: {
|
||||
projectId,
|
||||
agent: { agentId: autofillAgent.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if (!proxyType) {
|
||||
await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
generatePdfProxyActivity,
|
||||
transcodeVideoChunkActivity,
|
||||
deleteS3ObjectActivity,
|
||||
createAutofillTaskIfEnabledActivity,
|
||||
} from './transcode'
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
@@ -637,5 +638,55 @@ describe('Transcode Activities', () => {
|
||||
key: 'files/proj-123/a-audio-proxy.mp4',
|
||||
})
|
||||
})
|
||||
|
||||
it('should create ai_metadata_autofill task when autofill agent exists and asset is image', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Test User Autofill', email: 'test-autofill-agent@test.com' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team Autofill' },
|
||||
})
|
||||
await prisma.teamMember.create({
|
||||
data: { teamId: team.id, userId: user.id, role: 'owner' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Test Project Autofill', teamId: team.id },
|
||||
})
|
||||
const autofillAgent = await prisma.agent.create({
|
||||
data: {
|
||||
id: user.id,
|
||||
type: 'autofill',
|
||||
enabled: true,
|
||||
teamId: team.id,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
config: { provider: 'test', model: 'test' } as any,
|
||||
},
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'photo.png',
|
||||
mediaType: 'image/png',
|
||||
type: 'file',
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
media: { proxyType: 'image' } as any,
|
||||
},
|
||||
})
|
||||
|
||||
await createAutofillTaskIfEnabledActivity({
|
||||
assetId: asset.id,
|
||||
teamId: team.id,
|
||||
projectId: project.id,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: 'ai_metadata_autofill' },
|
||||
})
|
||||
expect(task).toBeDefined()
|
||||
expect(task?.status).toBe('pending')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -769,6 +769,82 @@ export async function createEmbeddingTaskIfEnabledActivity(
|
||||
})
|
||||
}
|
||||
|
||||
export interface CreateAutofillTaskIfEnabledParams {
|
||||
assetId: string
|
||||
teamId: string | null
|
||||
projectId: string | null
|
||||
}
|
||||
|
||||
export async function createAutofillTaskIfEnabledActivity(
|
||||
params: CreateAutofillTaskIfEnabledParams,
|
||||
): Promise<void> {
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: params.assetId },
|
||||
include: {
|
||||
project: true,
|
||||
},
|
||||
})
|
||||
if (!asset) {
|
||||
return
|
||||
}
|
||||
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
const isVideo = proxyType === 'video'
|
||||
const isImage = proxyType === 'image'
|
||||
if (!isVideo && !isImage) {
|
||||
return
|
||||
}
|
||||
|
||||
const teamId = params.teamId || asset.project?.teamId || null
|
||||
const projectId = params.projectId || asset.projectId
|
||||
|
||||
if (!teamId) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if there is an active autofill agent for the team
|
||||
const autofillAgent = await prisma.agent.findFirst({
|
||||
where: {
|
||||
type: 'autofill',
|
||||
enabled: true,
|
||||
user: { teamMembers: { some: { teamId } } },
|
||||
},
|
||||
})
|
||||
|
||||
if (!autofillAgent) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if a pending/processing autofill task already exists for this asset to avoid duplicates
|
||||
const existing = await prisma.workflowTask.findFirst({
|
||||
where: {
|
||||
assetId: params.assetId,
|
||||
type: WorkflowTaskType.ai_metadata_autofill,
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
},
|
||||
})
|
||||
if (existing) {
|
||||
return
|
||||
}
|
||||
|
||||
// Create autofill task
|
||||
await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: params.assetId,
|
||||
type: WorkflowTaskType.ai_metadata_autofill,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
teamId,
|
||||
projectId,
|
||||
payload: {
|
||||
projectId: projectId ?? '',
|
||||
agent: { agentId: autofillAgent.id },
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export interface TranscodeVideoChunkParams {
|
||||
assetId: string
|
||||
filePath: string
|
||||
|
||||
@@ -38,6 +38,9 @@ describe('transcodeImageWorkflow', () => {
|
||||
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'createEmbeddingTaskIfEnabledActivity',
|
||||
}),
|
||||
createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'createAutofillTaskIfEnabledActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -145,5 +148,11 @@ describe('transcodeImageWorkflow', () => {
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
})
|
||||
|
||||
expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-image',
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -23,6 +23,7 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void>
|
||||
updateAssetMediaActivity,
|
||||
downloadMediaToTmpActivity,
|
||||
createEmbeddingTaskIfEnabledActivity,
|
||||
createAutofillTaskIfEnabledActivity,
|
||||
} = getActivities()
|
||||
|
||||
await executeActivity(workerQueue, updateAssetStatusActivity, {
|
||||
@@ -96,6 +97,12 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void>
|
||||
projectId: task.projectId,
|
||||
})
|
||||
|
||||
await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, {
|
||||
assetId: asset.id,
|
||||
teamId: task.teamId,
|
||||
projectId: task.projectId,
|
||||
})
|
||||
|
||||
await completeTask(workerQueue, task.id)
|
||||
} catch (err) {
|
||||
console.error(`transcodeImageWorkflow failed for task ${task.id}:`, err)
|
||||
|
||||
@@ -41,6 +41,9 @@ describe('transcodePdfWorkflow', () => {
|
||||
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'createEmbeddingTaskIfEnabledActivity',
|
||||
}),
|
||||
createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'createAutofillTaskIfEnabledActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -150,5 +153,11 @@ describe('transcodePdfWorkflow', () => {
|
||||
assetId: 'asset-pdf',
|
||||
status: AssetStatus.processed,
|
||||
})
|
||||
|
||||
expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-pdf',
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -24,6 +24,7 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
|
||||
downloadMediaToTmpActivity,
|
||||
generatePdfProxyActivity,
|
||||
createEmbeddingTaskIfEnabledActivity,
|
||||
createAutofillTaskIfEnabledActivity,
|
||||
} = getActivities()
|
||||
|
||||
await executeActivity(workerQueue, updateAssetStatusActivity, {
|
||||
@@ -108,6 +109,12 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
|
||||
projectId: task.projectId,
|
||||
})
|
||||
|
||||
await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, {
|
||||
assetId: asset.id,
|
||||
teamId: task.teamId,
|
||||
projectId: task.projectId,
|
||||
})
|
||||
|
||||
await completeTask(workerQueue, task.id)
|
||||
} catch (err) {
|
||||
console.error(`transcodePdfWorkflow failed for task ${task.id}:`, err)
|
||||
|
||||
@@ -41,6 +41,9 @@ describe('transcodeVideoWorkflow', () => {
|
||||
createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'createEmbeddingTaskIfEnabledActivity',
|
||||
}),
|
||||
createAutofillTaskIfEnabledActivity: Object.assign(vi.fn(), {
|
||||
_activityName: 'createAutofillTaskIfEnabledActivity',
|
||||
}),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -151,6 +154,12 @@ describe('transcodeVideoWorkflow', () => {
|
||||
projectId: 'proj-1',
|
||||
})
|
||||
|
||||
expect(mockActivities.createAutofillTaskIfEnabledActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-1',
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
})
|
||||
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-1',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
|
||||
@@ -27,6 +27,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
updateAssetMediaActivity,
|
||||
downloadMediaToTmpActivity,
|
||||
createEmbeddingTaskIfEnabledActivity,
|
||||
createAutofillTaskIfEnabledActivity,
|
||||
} = getActivities()
|
||||
|
||||
await executeActivity(workerQueue, updateAssetStatusActivity, {
|
||||
@@ -161,6 +162,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
projectId: task.projectId,
|
||||
})
|
||||
|
||||
await executeActivity(workerQueue, createAutofillTaskIfEnabledActivity, {
|
||||
assetId: asset.id,
|
||||
teamId: task.teamId,
|
||||
projectId: task.projectId,
|
||||
})
|
||||
|
||||
await completeTask(workerQueue, task.id)
|
||||
} catch (err) {
|
||||
console.error(`transcodeVideoWorkflow failed for task ${task.id}:`, err)
|
||||
|
||||
Reference in New Issue
Block a user