diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a8d74a..3d6fdd0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +* **agent**: Fix an issue where 3D visual markup annotations in 1-on-1 chat sessions were inspected at 0° instead of the user's annotated degree angle * **transcode**: Fix an issue where portrait photos from cameras and phones appeared sideways in thumbnails, previews, watermarked share links and annotated snapshots, and were listed with their width and height swapped ### Changed diff --git a/packages/agent/src/activities/agent.test.ts b/packages/agent/src/activities/agent.test.ts index 1980151b..3575f69e 100644 --- a/packages/agent/src/activities/agent.test.ts +++ b/packages/agent/src/activities/agent.test.ts @@ -662,6 +662,23 @@ describe('Agent Activities', () => { const pdfPrompt = mockHarness.prompt.mock.calls[1]?.[0] ?? '' expect(pdfPrompt).toContain('Document pages: 50') expect(pdfPrompt).not.toContain('Video duration') + + // For a 3D asset, should produce 3D Model recommendation + await autofillAiActivity({ + teamId: 't1', + assetId: 'model-3', + assetName: 'robot.glb', + mediaType: 'model/gltf-binary', + images: [], + fields: [{ id: 'f1', config: { name: 'F1', type: 'text' } }], + context, + }) + + const modelPrompt = mockHarness.prompt.mock.calls[2]?.[0] ?? '' + expect(modelPrompt).toContain('3D Model: This asset has an orthogonal 4-view composite image') + expect(modelPrompt).toContain( + 'Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right', + ) }) }) @@ -1170,6 +1187,43 @@ describe('Agent Database Activities Integration', () => { expect((rootEntry as any)?.details?.thread).toBeUndefined() }) + it('should set media position to degree for 3D asset comments', async () => { + const asset3d = await prisma.asset.create({ + data: { + name: 'robot.glb', + type: 'file', + status: 'processed', + mediaType: 'model/gltf-binary', + media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo, + projectId: project.id, + }, + }) + + const comment3d = await prisma.assetComment.create({ + data: { + assetId: asset3d.id, + message: 'Nice angle', + creatorId: user.id, + second: 0.5, // 45° + }, + }) + + const sessionId = await initializeAgentSessionActivity({ + teamId: team.id, + agentId: 'test-agent-id', + userCommentId: comment3d.id, + userId: user.id, + }) + + const storage = new DatabaseSessionStorage(sessionId) + const pathEntries = await storage.getPathToRoot(comment3d.id) + const entry = pathEntries.find((e) => e.id === comment3d.id) + + expect(entry).toBeDefined() + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- test entry position parsing + expect((entry as any)?.details?.position).toEqual({ type: 'degree', degree: 45 }) + }) + it('should return user name and role for team member', async () => { const info = await getUserTeamInfoActivity({ userId: user.id, @@ -2105,5 +2159,83 @@ describe('Agent Database Activities Integration', () => { expect(systemPromptArg).toContain('') expect(systemPromptArg).toContain('read_thread') }) + + it('agentChatActivity injects # 3D Model Inspection into systemPrompt', async () => { + const user = await prisma.user.create({ + data: { name: 'User_3d_Prompt', email: `prompt-3d-${Date.now()}@example.com` }, + }) + const team = await prisma.team.create({ + data: { name: 'Team_3d_Prompt_' + Date.now() }, + }) + const project = await prisma.project.create({ + data: { name: 'Project_3d_Prompt', teamId: team.id }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'robot.glb', + type: 'file', + status: 'processed', + mediaType: 'model/gltf-binary', + media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo, + projectId: project.id, + }, + }) + + const comment = await prisma.assetComment.create({ + data: { + assetId: asset.id, + creatorId: user.id, + message: 'Check this 3d model', + }, + }) + + const mockSession = { + id: 'mock-session-id', + subscribe: vi.fn(), + getEntries: vi.fn().mockResolvedValue([]), + getStorage: vi.fn().mockReturnValue({ sessionId: 'mock-session-id' }), + } + const mockHarness = { + subscribe: vi.fn(), + prompt: vi.fn().mockResolvedValue({ + content: [{ type: 'text', text: 'Response' }], + usage: { input: 10, output: 20 }, + }), + } + + vi.mocked(piAgent.createAgentSession).mockResolvedValue({ + session: mockSession as unknown as Session, + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock AgentHarness instance for activity test + harness: mockHarness as unknown as AgentHarness, + }) + + const context = { + agent: { id: 'agent-1', provider: { name: 'google' }, modelRef: { modelId: 'gemini' } }, + dbProviders: [], + teamSkills: [], + allowedDomains: [], + } as unknown as AgentExecutionContext + + await agentChatActivity({ + teamId: team.id, + agentId: 'agent-1', + message: 'Hello', + imageUrls: [], + projectId: project.id, + folderId: asset.id, + assetId: asset.id, + sessionId: 'mock-session-id', + userCommentId: comment.id, + userId: user.id, + context, + }) + + const lastCall = vi.mocked(piAgent.createAgentSession).mock.calls.at(-1) + const systemPromptArg = lastCall?.[0].systemPrompt as string + expect(systemPromptArg).toContain('# 3D Model Inspection') + expect(systemPromptArg).toContain('media_type="3d"') + expect(systemPromptArg).toContain('config3d') + expect(systemPromptArg).toContain('') + }) }) }) diff --git a/packages/agent/src/activities/agent.ts b/packages/agent/src/activities/agent.ts index 6942d726..bf03c10f 100644 --- a/packages/agent/src/activities/agent.ts +++ b/packages/agent/src/activities/agent.ts @@ -21,6 +21,7 @@ import { import { aiUsageService } from '@shumai/core/src/ai-usage/ai-usage' import { quotaService, QuotaExceededError } from '@shumai/core/src/quota/quota-service' import { getProxyType } from '@shumai/core/src/utils/mime' +import { secondToDegree } from '@shumai/core/src/utils/model3d' import { UpdateAssetMetadataRequest, type ShumaiMessageContext, @@ -179,6 +180,8 @@ User messages may contain a block detailing the user, active asset loc systemPrompt += `\n\n# Comment Threads\nWhen operating in comment mode, previous top-level messages in the conversation history may contain a tag indicating an earlier discussion thread with replies. If a user's question refers to or depends on earlier comments or discussions, you can use the 'read_thread' tool with the thread ID to inspect all replies in that thread.` } + systemPrompt += `\n\n# 3D Model Inspection\nWhen interacting with 3D model assets (media_type="3d"), you can visually inspect the asset using 'read_asset'. Calling 'read_asset' with config3d: null (or config3d: {}) returns an orthogonal 4-view composite image (Front at top-left, Right side at top-right, Back at bottom-left, Left side at bottom-right). If a tag is present or a specific angle is discussed, pass 'config3d: { degree: ... }' (integer 0, 15, 30, ..., 345) to inspect that view angle. If an tag is present on a 3D asset, pass 'annotationId' to inspect the angle with the user's visual markup drawings overlaid.` + if (agent.type === 'autofill') { systemPrompt += `\n\n# Autofill Instructions\nWhen performing metadata autofill, inspect the asset content using the 'read_asset' tool. If 'read_asset' fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.` } @@ -564,6 +567,7 @@ export async function autofillAiActivity(params: AutofillAiParams) { const proxyType = getProxyType(params.mediaType, params.assetName) const isVideoOrAudio = proxyType === 'video' || proxyType === 'audio' const isPdf = proxyType === 'pdf' + const is3d = proxyType === '3d' if (params.duration !== undefined && params.duration > 0 && (!proxyType || isVideoOrAudio)) { promptLines.push( @@ -575,11 +579,16 @@ export async function autofillAiActivity(params: AutofillAiParams) { `- Document pages: ${params.pageCount}. Recommended inspection: call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ${Math.min(params.pageCount, 20)} } or mode: "text".`, ) } + if (is3d) { + promptLines.push( + '- 3D Model: This asset has an orthogonal 4-view composite image (2x2 grid showing Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right). Call read_asset to inspect the 4 views.', + ) + } promptLines.push( '', 'Instructions:', - `1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type).`, + `1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type; for 3D assets, call read_asset directly to inspect the 4-view composite image).`, '2. After analyzing the retrieved content, call the "autofill_metadata" tool to provide the extracted metadata values.', '3. If "read_asset" fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.', ) @@ -834,7 +843,9 @@ export async function initializeAgentSessionActivity(params: { let position: ShumaiMediaPosition | undefined = undefined if (c.second !== null && c.second !== undefined) { - if (proxyType === 'pdf') { + if (proxyType === '3d') { + position = { type: 'degree', degree: secondToDegree(c.second) } + } else if (proxyType === 'pdf') { position = { type: 'page', page: Math.round(c.second) } } else { position = { type: 'time', seconds: c.second } @@ -852,7 +863,14 @@ export async function initializeAgentSessionActivity(params: { id: att.asset.id, name: att.asset.name, type: att.asset.type, - mediaType: attProxyType as 'image' | 'video' | 'pdf' | 'audio' | 'other' | undefined, + mediaType: attProxyType as + | 'image' + | 'video' + | 'pdf' + | 'audio' + | '3d' + | 'other' + | undefined, mimeType: att.asset.mediaType || undefined, }) } diff --git a/packages/agent/src/context/serialize-context.test.ts b/packages/agent/src/context/serialize-context.test.ts index 1c72c542..1811fcf6 100644 --- a/packages/agent/src/context/serialize-context.test.ts +++ b/packages/agent/src/context/serialize-context.test.ts @@ -350,5 +350,24 @@ describe('serializeContextToXml', () => { '\n \n', ) }) + + it('serializes position type="degree" with integer degree attribute without seconds', () => { + const context: ShumaiMessageContext = { + currentAsset: { + id: 'ast_3d', + name: 'robot.glb', + type: 'file', + mediaType: '3d', + path: 'robot.glb', + }, + position: { + type: 'degree', + degree: 45, + }, + } + expect(serializeContextToXml(context)).toBe( + '\n \n \n', + ) + }) }) }) diff --git a/packages/agent/src/tools/annotation-resolver.test.ts b/packages/agent/src/tools/annotation-resolver.test.ts index 2ea0e0df..5452944b 100644 --- a/packages/agent/src/tools/annotation-resolver.test.ts +++ b/packages/agent/src/tools/annotation-resolver.test.ts @@ -4,6 +4,7 @@ import { setupTestDbHooks } from '@shumai/db/test' import { resolveAnnotationsById } from './annotation-resolver' import { DatabaseSessionStorage } from '../database-session-storage' import type { SessionTreeEntry } from '@earendil-works/pi-agent-core' +import { degreeToSecond } from '@shumai/core/src/utils/model3d' describe('resolveAnnotationsById', () => { setupTestDbHooks() @@ -210,6 +211,89 @@ describe('resolveAnnotationsById', () => { ]) }) + it('resolves annotations and degree position timestamp from 1-on-1 chat AgentSessionEntry for 3D assets', async () => { + const user = await prisma.user.create({ + data: { name: 'User 3D', email: 'user3d@test.com', password: 'pw' }, + }) + const team = await prisma.team.create({ + data: { name: 'Team 3D' }, + }) + const project = await prisma.project.create({ + data: { name: 'Project 3D', teamId: team.id }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'robot.glb', + type: 'file', + projectId: project.id, + status: 'uploaded', + }, + }) + const agentUser = await prisma.user.create({ + data: { name: 'Agent 3D', email: 'agent3d@example.com', type: 'agent' }, + }) + const agent = await prisma.agent.create({ + data: { + id: agentUser.id, + teamId: team.id, + type: 'chat', + config: { provider: 'test-provider', model: 'test-model' }, + }, + }) + + const storage = await DatabaseSessionStorage.create({ + agentId: agent.id, + userId: user.id, + assetId: asset.id, + }) + + storage.currentMessageContext = { + currentAsset: { + id: asset.id, + name: asset.name, + type: 'file', + path: asset.name, + }, + position: { type: 'degree', degree: 45 }, + annotation: true, + annotations: [ + { + type: 'rectangle', + x: 10, + y: 20, + width: 30, + height: 40, + }, + ], + } + + const userEntry: SessionTreeEntry = { + type: 'message', + id: '01JTEST3DENTRY123456789', + parentId: null, + timestamp: new Date().toISOString(), + message: { + role: 'user', + content: 'Check markup at 45 degrees', + timestamp: Date.now(), + }, + } + + await storage.appendEntry(userEntry) + + const result = await resolveAnnotationsById(asset.id, '01JTEST3DENTRY123456789') + expect(result.timestamp).toBeCloseTo(degreeToSecond(45)) + expect(result.annotations).toEqual([ + { + type: 'rectangle', + x: 10, + y: 20, + width: 30, + height: 40, + }, + ]) + }) + it('throws an error when AgentSessionEntry belongs to a different asset', async () => { const user = await prisma.user.create({ data: { name: 'User', email: 'user3@test.com', password: 'pw' }, diff --git a/packages/agent/src/tools/annotation-resolver.ts b/packages/agent/src/tools/annotation-resolver.ts index 97128e1f..0f928614 100644 --- a/packages/agent/src/tools/annotation-resolver.ts +++ b/packages/agent/src/tools/annotation-resolver.ts @@ -1,5 +1,6 @@ import { prisma } from '@shumai/db' import type { ShumaiMessageContext } from '@shumai/dtos' +import { degreeToSecond } from '@shumai/core/src/utils/model3d' export async function resolveAnnotationsById( assetId: string, @@ -44,7 +45,14 @@ export async function resolveAnnotationsById( const list = details?.annotations const annotations = Array.isArray(list) && list.length > 0 ? (list as unknown as PrismaJson.AnnotationList) : null - const timestamp = details?.position?.type === 'time' ? details.position.seconds : null + let timestamp: number | null = null + if (details?.position?.type === 'time') { + timestamp = details.position.seconds + } else if (details?.position?.type === 'degree') { + timestamp = degreeToSecond(details.position.degree) + } else if (details?.position?.type === 'page') { + timestamp = details.position.page + } return { annotations, timestamp } } diff --git a/packages/agent/src/tools/read-asset.test.ts b/packages/agent/src/tools/read-asset.test.ts index a9b86a68..1073e640 100644 --- a/packages/agent/src/tools/read-asset.test.ts +++ b/packages/agent/src/tools/read-asset.test.ts @@ -11,6 +11,7 @@ import { s3Service } from '@shumai/core/src/s3/s3' import { workflowService } from '@shumai/workflow-core' import { authzService } from '@shumai/core/src/authz/authz' import * as annotationResolver from './annotation-resolver' +import { degreeToSecond } from '@shumai/core/src/utils/model3d' vi.mock('@shumai/db', async (importOriginal) => { const actual = await importOriginal() @@ -211,6 +212,476 @@ describe('readAssetTool', () => { }) }) + describe('3D Assets', () => { + it('should read 3D asset 4-views image and return image with quadrant layout description', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + storageKey: { key: 'raw/robot.glb' }, + media: { + proxyType: '3d', + imageTranscodes: [{ key: 'proxy/views.webp' }], + }, + } as unknown as Asset) + + vi.mocked(s3Service.getObject).mockResolvedValue({ + buffer: Buffer.from('fake-views-bytes'), + contentType: 'image/webp', + } as unknown as { buffer: Buffer; contentType: string }) + + const tool = createReadAssetTool('user-1') + const result = await tool.execute('call-3d', { + assetId: 'model-1', + annotationId: null, + imageConfig: null, + videoConfig: null, + docConfig: null, + }) + + expect(authzService.hasPermission).toHaveBeenCalledWith({ + user: { id: 'user-1' }, + permission: 'Read', + type: 'asset', + id: 'model-1', + }) + expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/views.webp') + + const textItem = result.content[0] as { type: 'text'; text: string } + const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string } + + expect(textItem.text).toContain('Orthogonal 4-view composite image') + expect(textItem.text).toContain('Top-Left quadrant: Front view (0°)') + expect(textItem.text).toContain('Top-Right quadrant: Right side view (90°)') + expect(textItem.text).toContain('Bottom-Left quadrant: Back view (180°)') + expect(textItem.text).toContain('Bottom-Right quadrant: Left side view (270°)') + expect(imageItem.type).toBe('image') + expect(imageItem.data).toBe(Buffer.from('fake-views-bytes').toString('base64')) + expect(result.details.key).toBe('proxy/views.webp') + }) + + it('should support s3KeyOnly mode for 3D assets', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + media: { + proxyType: '3d', + imageTranscodes: [{ key: 'proxy/views.webp' }], + }, + } as unknown as Asset) + + const tool = createReadAssetTool('user-1') + const result = await tool.execute('call-3d-s3', { + assetId: 'model-1', + annotationId: null, + s3KeyOnly: true, + imageConfig: null, + videoConfig: null, + docConfig: null, + }) + + expect(result.content).toHaveLength(1) + const textItem = result.content[0] as { type: 'text'; text: string } + expect(textItem.text).toContain('proxy/views.webp') + expect(textItem.text).toContain('Front view (0°)') + expect(result.details.key).toBe('proxy/views.webp') + }) + + it('should throw error if videoConfig is passed for 3D assets', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + media: { + proxyType: '3d', + imageTranscodes: [{ key: 'proxy/views.webp' }], + }, + } as unknown as Asset) + + const tool = createReadAssetTool('user-1') + await expect( + tool.execute('call-3d-err', { + assetId: 'model-1', + annotationId: null, + imageConfig: null, + videoConfig: { start: 0, end: 4, count: 4 }, + docConfig: null, + }), + ).rejects.toThrow( + "To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig", + ) + }) + + it('should throw error if 3D asset imageTranscodes is missing when overview is requested', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + media: { + proxyType: '3d', + imageTranscodes: [], + }, + } as unknown as Asset) + + const tool = createReadAssetTool('user-1') + await expect( + tool.execute('call-3d-err2', { + assetId: 'model-1', + annotationId: null, + imageConfig: null, + videoConfig: null, + docConfig: null, + }), + ).rejects.toThrow('4-views image (views.webp) is missing') + }) + + it('should extract specific turntable frame when config3d.degree is provided', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + projectId: 'proj-1', + media: { + proxyType: '3d', + videoTranscodes: [{ key: 'proxy/turntable.mp4' }], + imageTranscodes: [{ key: 'proxy/views.webp' }], + }, + } as unknown as Asset) + + vi.mocked(prisma.workflowTask.create).mockResolvedValue({ + id: 'task-shot-3d', + } as unknown as WorkflowTask) + + vi.mocked(workflowService.executeWait).mockResolvedValue({ + id: 'task-shot-3d', + status: WorkflowTaskStatus.completed, + output: { + screenshots: [{ key: 'proxy/shot-45.webp', timestamp: 0.5833 }], + }, + } as unknown as WorkflowTask) + + vi.mocked(s3Service.getObject).mockResolvedValue({ + buffer: Buffer.from('fake-shot-45-bytes'), + contentType: 'image/webp', + } as unknown as { buffer: Buffer; contentType: string }) + + const tool = createReadAssetTool('user-1') + const result = await tool.execute('call-3d-degree', { + assetId: 'model-1', + annotationId: null, + config3d: { degree: 45 }, + imageConfig: null, + videoConfig: null, + docConfig: null, + }) + + expect(prisma.workflowTask.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + type: WorkflowTaskType.transcode_screenshot, + assetId: 'model-1', + payload: { + projectId: 'proj-1', + screenshot: { + start: expect.any(Number), + end: expect.any(Number), + count: 1, + commentTimestamp: expect.any(Number), + annotations: undefined, + }, + }, + }), + }) + + const textItem = result.content[0] as { type: 'text'; text: string } + const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string } + + expect(textItem.text).toContain('view at 45°:') + expect(imageItem.data).toBe(Buffer.from('fake-shot-45-bytes').toString('base64')) + expect(result.details.key).toBe('proxy/shot-45.webp') + }) + + it('should throw error when config3d.degree is not a multiple of 15 or out of range', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + media: { + proxyType: '3d', + videoTranscodes: [{ key: 'proxy/turntable.mp4' }], + }, + } as unknown as Asset) + + const tool = createReadAssetTool('user-1') + + await expect( + tool.execute('call-3d-invalid-deg', { + assetId: 'model-1', + annotationId: null, + config3d: { degree: 40 }, + imageConfig: null, + videoConfig: null, + docConfig: null, + }), + ).rejects.toThrow('Invalid turntable degree: 40') + + await expect( + tool.execute('call-3d-out-of-range', { + assetId: 'model-1', + annotationId: null, + config3d: { degree: 360 }, + imageConfig: null, + videoConfig: null, + docConfig: null, + }), + ).rejects.toThrow('Invalid turntable degree: 360') + }) + + it('should extract frame with overlaid markups when annotationId is provided on 3D asset', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + projectId: 'proj-1', + media: { + proxyType: '3d', + videoTranscodes: [{ key: 'proxy/turntable.mp4' }], + }, + } as unknown as Asset) + + vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({ + annotations: [ + { + type: 'freehand', + color: '#ff0000', + points: [ + [0, 0], + [10, 10], + ], + }, + ], + timestamp: 0.5, // 0.5s = 45° + }) + + vi.mocked(prisma.workflowTask.create).mockResolvedValue({ + id: 'task-shot-ann', + } as unknown as WorkflowTask) + + vi.mocked(workflowService.executeWait).mockResolvedValue({ + id: 'task-shot-ann', + status: WorkflowTaskStatus.completed, + output: { + screenshots: [{ key: 'proxy/shot-annotated.webp', timestamp: 0.5 }], + }, + } as unknown as WorkflowTask) + + vi.mocked(s3Service.getObject).mockResolvedValue({ + buffer: Buffer.from('fake-shot-ann-bytes'), + contentType: 'image/webp', + } as unknown as { buffer: Buffer; contentType: string }) + + const tool = createReadAssetTool('user-1') + const result = await tool.execute('call-3d-ann', { + assetId: 'model-1', + annotationId: 'comm-1', + config3d: null, + imageConfig: null, + videoConfig: null, + docConfig: null, + }) + + expect(prisma.workflowTask.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + type: WorkflowTaskType.transcode_screenshot, + assetId: 'model-1', + payload: { + projectId: 'proj-1', + screenshot: { + start: 0.5, + end: 0.5, + count: 1, + commentTimestamp: 0.5, + annotations: [ + { + type: 'freehand', + color: '#ff0000', + points: [ + [0, 0], + [10, 10], + ], + }, + ], + }, + }, + }), + }) + + const textItem = result.content[0] as { type: 'text'; text: string } + expect(textItem.text).toContain('view at 45° with visual markups overlaid:') + expect(result.details.key).toBe('proxy/shot-annotated.webp') + }) + + it('should extract turntable frame at resolved degree angle for 3D session annotations', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + projectId: 'proj-1', + media: { + proxyType: '3d', + videoTranscodes: [{ key: 'proxy/turntable.mp4' }], + }, + } as unknown as Asset) + + const expectedTime = degreeToSecond(90) + vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({ + annotations: [ + { + type: 'box', + color: '#00ff00', + points: [ + [10, 20], + [40, 60], + ], + }, + ], + timestamp: expectedTime, + }) + + vi.mocked(prisma.workflowTask.create).mockResolvedValue({ + id: 'task-shot-3d-session', + } as unknown as WorkflowTask) + + vi.mocked(workflowService.executeWait).mockResolvedValue({ + id: 'task-shot-3d-session', + status: WorkflowTaskStatus.completed, + output: { + screenshots: [{ key: 'proxy/shot-annotated-90.webp', timestamp: expectedTime }], + }, + } as unknown as WorkflowTask) + + vi.mocked(s3Service.getObject).mockResolvedValue({ + buffer: Buffer.from('fake-shot-90-bytes'), + contentType: 'image/webp', + } as unknown as { buffer: Buffer; contentType: string }) + + const tool = createReadAssetTool('user-1') + const result = await tool.execute('call-3d-session-ann', { + assetId: 'model-1', + annotationId: 'session-entry-1', + config3d: null, + imageConfig: null, + videoConfig: null, + docConfig: null, + }) + + expect(prisma.workflowTask.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + type: WorkflowTaskType.transcode_screenshot, + assetId: 'model-1', + payload: { + projectId: 'proj-1', + screenshot: { + start: expectedTime, + end: expectedTime, + count: 1, + commentTimestamp: expectedTime, + annotations: [ + { + type: 'box', + color: '#00ff00', + points: [ + [10, 20], + [40, 60], + ], + }, + ], + }, + }, + }), + }) + + const textItem = result.content[0] as { type: 'text'; text: string } + expect(textItem.text).toContain('view at 90° with visual markups overlaid:') + expect(result.details.key).toBe('proxy/shot-annotated-90.webp') + }) + + it('should throw error if config3d.degree conflicts with annotation angle', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + projectId: 'proj-1', + media: { + proxyType: '3d', + videoTranscodes: [{ key: 'proxy/turntable.mp4' }], + }, + } as unknown as Asset) + + vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({ + annotations: [ + { + type: 'box', + color: '#ff0000', + points: [ + [0, 0], + [10, 10], + ], + }, + ], + timestamp: 0.5, // 45° + }) + + const tool = createReadAssetTool('user-1') + await expect( + tool.execute('call-3d-conflict', { + assetId: 'model-1', + annotationId: 'comm-1', + config3d: { degree: 90 }, // Conflicts with 45° + imageConfig: null, + videoConfig: null, + docConfig: null, + }), + ).rejects.toThrow( + 'Annotation "comm-1" was drawn at 45°, but degree 90° was specified. Do not pass a conflicting degree with annotationId.', + ) + }) + + it('should throw error if 3D asset turntable video is missing when angle inspection is requested', async () => { + vi.mocked(authzService.hasPermission).mockResolvedValue() + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'model-1', + name: 'robot.glb', + mediaType: 'model/gltf-binary', + media: { + proxyType: '3d', + videoTranscodes: [], + }, + } as unknown as Asset) + + const tool = createReadAssetTool('user-1') + await expect( + tool.execute('call-3d-missing-turntable', { + assetId: 'model-1', + annotationId: null, + config3d: { degree: 45 }, + imageConfig: null, + videoConfig: null, + docConfig: null, + }), + ).rejects.toThrow('turntable video (turntable.mp4) is missing') + }) + }) + describe('Video Assets', () => { it('should throw error if videoConfig is missing for video assets', async () => { vi.mocked(authzService.hasPermission).mockResolvedValue() diff --git a/packages/agent/src/tools/read-asset.ts b/packages/agent/src/tools/read-asset.ts index 9328a79a..b7f09b6c 100644 --- a/packages/agent/src/tools/read-asset.ts +++ b/packages/agent/src/tools/read-asset.ts @@ -7,6 +7,12 @@ import { workflowService } from '@shumai/workflow-core' import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz' import { resolveAnnotationsById } from './annotation-resolver' import { getFileMimeType } from '@shumai/core/src/utils/file-mime' +import { is3dModel } from '@shumai/core/src/utils/mime' +import { + isValidTurntableDegree, + secondToDegree, + degreeToSecond, +} from '@shumai/core/src/utils/model3d' export const readAssetSchema = Type.Object( { @@ -37,6 +43,31 @@ export const readAssetSchema = Type.Object( ), Type.Null(), ]), + config3d: Type.Optional( + Type.Union([ + Type.Object( + { + degree: Type.Optional( + Type.Union([ + Type.Integer({ + minimum: 0, + maximum: 345, + description: + 'Turntable rotation degree (integer: 0, 15, 30, ..., 345 in 15° increments) to inspect a specific angle of a 3D model. Leave null/omitted to view the 4-view composite overview.', + }), + Type.Null(), + ]), + ), + }, + { + additionalProperties: false, + description: + 'Configuration for 3D model assets. Pass null if inspecting another asset type. Optionally pass degree to inspect a specific turntable angle, or leave null/omitted to view the 4-view composite overview.', + }, + ), + Type.Null(), + ]), + ), videoConfig: Type.Union([ Type.Object( { @@ -167,12 +198,12 @@ export function createReadAssetTool( label: 'Read Asset', description: 'Inspects and reads the content of an asset in the workspace. ' + - 'Supports visual analysis for images, video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' + - 'Pass assetId, and provide the specific config (imageConfig, videoConfig, or docConfig) while setting unused configs to null. ' + + 'Supports visual analysis for images, 3D models (overview or specific degree angle), video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' + + 'Pass assetId, and provide the specific config (imageConfig, config3d, videoConfig, or docConfig) while setting unused configs to null. ' + 'Optionally provide annotationId to view visual markups drawn on the asset.', parameters: readAssetSchema, execute: async (_toolCallId, params) => { - const { assetId, annotationId, s3KeyOnly, videoConfig, docConfig } = params + const { assetId, annotationId, s3KeyOnly, config3d, videoConfig, docConfig } = params if (agentType === 'autofill' && !userId) { if (!targetAssetId) { @@ -243,6 +274,209 @@ export function createReadAssetTool( const filename = asset.name?.toLowerCase() || '' const bucket = process.env.S3_BUCKET || 'shumai' + // ---------------------------------------------------------------------- + // Branch 0: 3D Assets (Orthogonal 4-View Composite Image or Degree Angle) + // ---------------------------------------------------------------------- + const is3d = proxyType === '3d' || is3dModel(mediaType, filename) + if (is3d) { + if (videoConfig) { + throw new Error( + `Asset ${asset.id} is a 3D model. To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig.`, + ) + } + + if (config3d?.degree !== undefined && config3d?.degree !== null) { + if (!isValidTurntableDegree(config3d.degree)) { + throw new Error( + `Invalid turntable degree: ${config3d.degree}. Degree must be an integer multiple of 15 between 0 and 345 (e.g. 0, 15, 30, 45, ..., 345).`, + ) + } + } + + let annotations: PrismaJson.AnnotationList | null = null + let commentTimestamp: number | null = null + if (annotationId) { + const resolved = await resolveAnnotationsById(asset.id, annotationId) + annotations = resolved.annotations + commentTimestamp = resolved.timestamp + if (commentTimestamp !== null && commentTimestamp !== undefined) { + const commentDegree = secondToDegree(commentTimestamp) + if ( + config3d?.degree !== undefined && + config3d?.degree !== null && + config3d.degree !== commentDegree + ) { + throw new Error( + `Annotation "${annotationId}" was drawn at ${commentDegree}°, but degree ${config3d.degree}° was specified. Do not pass a conflicting degree with annotationId.`, + ) + } + } + } + + if ((config3d?.degree !== undefined && config3d?.degree !== null) || annotationId) { + if (!mediaInfo?.videoTranscodes || mediaInfo.videoTranscodes.length === 0) { + throw new Error( + `Asset ${asset.id} is a 3D model, but its turntable video (turntable.mp4) is missing.`, + ) + } + + let targetDegree: number + let targetSecond: number + + if (config3d?.degree !== undefined && config3d?.degree !== null) { + targetDegree = config3d.degree + targetSecond = degreeToSecond(targetDegree) + } else { + targetSecond = + commentTimestamp !== null && commentTimestamp !== undefined ? commentTimestamp : 0 + targetDegree = secondToDegree(targetSecond) + } + + const task = await prisma.workflowTask.create({ + data: { + assetId: asset.id, + projectId: asset.projectId || 'none', + type: WorkflowTaskType.transcode_screenshot, + status: WorkflowTaskStatus.pending, + payload: { + projectId: asset.projectId || 'none', + screenshot: { + start: targetSecond, + end: targetSecond, + count: 1, + commentTimestamp: targetSecond, + annotations: annotations ?? undefined, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode screenshot output + const output = completedTask.output as any + const screenshots = output?.screenshots as + | Array<{ key: string; timestamp: number }> + | undefined + + if (!screenshots || screenshots.length === 0) { + throw new Error( + `Failed to extract 3D turntable frame for asset ${asset.id} at ${targetDegree}°.`, + ) + } + + const shotKey = screenshots[0].key + const downloadUrl = await s3Service.presign( + bucket, + shotKey, + 'GET', + true, + asset.name || undefined, + ) + + const content: Array = [] + const hasMarkup = Boolean(annotations && annotations.length > 0) + const textDescription = hasMarkup + ? `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}° with visual markups overlaid:` + : `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}°:` + + if (s3KeyOnly) { + content.push({ + type: 'text', + text: textDescription, + }) + } else { + const { buffer, contentType } = await s3Service.getObject(bucket, shotKey) + const actualMimeType = + contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp' + content.push({ + type: 'text', + text: textDescription, + }) + content.push({ + type: 'image', + data: buffer.toString('base64'), + mimeType: actualMimeType, + }) + } + + return { + content, + details: { + assetId: asset.id, + name: asset.name, + mediaType: asset.mediaType, + key: shotKey, + downloadUrl, + sourceKeys: [shotKey], + }, + } + } + + if (!mediaInfo?.imageTranscodes || mediaInfo.imageTranscodes.length === 0) { + throw new Error( + `Asset ${asset.id} is a 3D model, but its 4-views image (views.webp) is missing.`, + ) + } + + const mediaKey = mediaInfo.imageTranscodes[0].key + if (!mediaKey) { + throw new Error(`Asset ${asset.id} is a 3D model, but its 4-views image key is missing.`) + } + + const keyToUse = mediaKey + const mimeType = 'image/webp' + + const downloadUrl = await s3Service.presign( + bucket, + keyToUse, + 'GET', + true, + asset.name || undefined, + ) + + const content: Array = [] + const textDescription = [ + `3D asset "${asset.name}" (ID: ${asset.id}, MIME: ${mimeType}, S3 Key: "${keyToUse}"):`, + 'Orthogonal 4-view composite image (2x2 grid across 360° rotation):', + '- Top-Left quadrant: Front view (0°)', + '- Top-Right quadrant: Right side view (90°)', + '- Bottom-Left quadrant: Back view (180°)', + '- Bottom-Right quadrant: Left side view (270°)', + ].join('\n') + + if (s3KeyOnly) { + content.push({ + type: 'text', + text: textDescription, + }) + } else { + const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse) + const actualMimeType = + contentType && contentType !== 'application/octet-stream' ? contentType : mimeType + content.push({ + type: 'text', + text: textDescription, + }) + content.push({ + type: 'image', + data: buffer.toString('base64'), + mimeType: actualMimeType, + }) + } + + return { + content, + details: { + assetId: asset.id, + name: asset.name, + mediaType: asset.mediaType, + key: keyToUse, + downloadUrl, + sourceKeys: [keyToUse], + }, + } + } + // ---------------------------------------------------------------------- // Branch 1: Image Assets // ---------------------------------------------------------------------- diff --git a/packages/agent/src/tools/read-thread.test.ts b/packages/agent/src/tools/read-thread.test.ts index a4fbb6f3..51e20346 100644 --- a/packages/agent/src/tools/read-thread.test.ts +++ b/packages/agent/src/tools/read-thread.test.ts @@ -129,6 +129,42 @@ describe('createReadThreadTool', () => { expect(textContent).toContain('- [Ai Agent] (id: reply-1, has_markup: false): Agent answer') }) + it('should format 3d asset comment timestamps as degree: X°', async () => { + vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({ + id: 'root-3d', + message: 'Notice this angle', + creator: { name: 'Alice' }, + second: 0.5, + annotation: [{ type: 'box' }], + asset: { + media: { proxyType: '3d' }, + mediaType: 'model/gltf-binary', + name: 'robot.glb', + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any) + vi.mocked(prisma.assetComment.findMany).mockResolvedValue([ + { + id: 'reply-3d-1', + message: 'Looking at back side', + creator: { name: 'Bob' }, + second: 2.0, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ] as any) + + const tool = createReadThreadTool() + const result = await tool.execute('call-3d', { threadId: 'root-3d' }) + const textContent = (result.content[0] as { text: string }).text + + expect(textContent).toContain( + 'Thread Root [Alice] (id: root-3d, degree: 45°, has_markup: true): Notice this angle', + ) + expect(textContent).toContain( + '- [Bob] (id: reply-3d-1, degree: 180°, has_markup: false): Looking at back side', + ) + }) + it('should describe threadId parameter as referencing in ', () => { const tool = createReadThreadTool() // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/agent/src/tools/read-thread.ts b/packages/agent/src/tools/read-thread.ts index 4a258c95..f3c22986 100644 --- a/packages/agent/src/tools/read-thread.ts +++ b/packages/agent/src/tools/read-thread.ts @@ -1,6 +1,8 @@ import { prisma } from '@shumai/db' import { Type } from 'typebox' import { type AgentTool } from '@earendil-works/pi-agent-core' +import { secondToDegree } from '@shumai/core/src/utils/model3d' +import { getProxyType } from '@shumai/core/src/utils/mime' const readThreadSchema = Type.Object({ threadId: Type.String({ @@ -9,14 +11,21 @@ const readThreadSchema = Type.Object({ }), }) -function formatCommentMeta(c: { - id: string - second?: number | null - annotation?: unknown -}): string { +function formatCommentMeta( + c: { + id: string + second?: number | null + annotation?: unknown + }, + proxyType?: string, +): string { const parts = [`id: ${c.id}`] if (c.second !== null && c.second !== undefined) { - parts.push(`time: ${c.second}s`) + if (proxyType === '3d') { + parts.push(`degree: ${secondToDegree(c.second)}°`) + } else { + parts.push(`time: ${c.second}s`) + } } const hasMarkup = Boolean(c.annotation && Array.isArray(c.annotation) && c.annotation.length > 0) parts.push(`has_markup: ${hasMarkup}`) @@ -34,7 +43,10 @@ export const createReadThreadTool = (): AgentTool< execute: async (_toolCallId, params) => { const rootComment = await prisma.assetComment.findUnique({ where: { id: params.threadId }, - include: { creator: true }, + include: { + creator: true, + asset: { select: { media: true, mediaType: true, name: true } }, + }, }) if (!rootComment) { @@ -61,9 +73,14 @@ export const createReadThreadTool = (): AgentTool< include: { creator: true }, }) + const proxyType = + (rootComment.asset?.media as PrismaJson.MediaInfo | null)?.proxyType || + getProxyType(rootComment.asset?.mediaType, rootComment.asset?.name) || + undefined + const isRootAgent = rootComment.creator?.type === 'agent' || rootComment.sessionId !== null const rootAuthor = rootComment.creator?.name || (isRootAgent ? 'Ai Agent' : 'User') - const rootMeta = formatCommentMeta(rootComment) + const rootMeta = formatCommentMeta(rootComment, proxyType) let output = `Thread Root [${rootAuthor}] ${rootMeta}: ${rootComment.message || ''}\n\nReplies (${replies.length}):` if (replies.length === 0) { @@ -72,7 +89,7 @@ export const createReadThreadTool = (): AgentTool< for (const reply of replies) { const isAgent = reply.creator?.type === 'agent' || reply.sessionId !== null const author = reply.creator?.name || (isAgent ? 'Ai Agent' : 'User') - const replyMeta = formatCommentMeta(reply) + const replyMeta = formatCommentMeta(reply, proxyType) output += `\n- [${author}] ${replyMeta}: ${reply.message || ''}` } } diff --git a/packages/agent/src/workflows/agent-chat.test.ts b/packages/agent/src/workflows/agent-chat.test.ts index 39a90531..7d41b7ef 100644 --- a/packages/agent/src/workflows/agent-chat.test.ts +++ b/packages/agent/src/workflows/agent-chat.test.ts @@ -806,4 +806,89 @@ describe('Agent Chat Workflow', () => { }), ) }) + + it('should format mediaPosition as degree for 3D assets in 1-on-1 chat', async () => { + mockActivities.getAssetActivity.mockResolvedValue({ + id: 'a-3d', + project: { teamId: 't1' }, + type: 'file', + name: 'model.glb', + mediaType: 'model/gltf-binary', + media: { proxyType: '3d' }, + parentId: null, + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'chat', + status: 'pending', + assetId: 'a-3d', + payload: { + projectId: 'p1', + agent: { + agentId: 'b1', + prompt: 'Look at this angle', + sessionId: 'session-3d-1', + second: 0.5, // 0.5s turntable time = 45° + }, + }, + }, + }) + + await agentChat(task) + + expect(mockActivities.agentChatActivity).toHaveBeenCalledWith( + expect.objectContaining({ + messageContext: expect.objectContaining({ + position: { type: 'degree', degree: 45 }, + }), + }), + ) + }) + + it('should format mediaPosition as degree for 3D assets in comment mode', async () => { + mockActivities.getAssetActivity.mockResolvedValue({ + id: 'a-3d', + project: { teamId: 't1' }, + type: 'file', + name: 'model.glb', + mediaType: 'model/gltf-binary', + media: { proxyType: '3d' }, + parentId: null, + }) + + mockActivities.getCommentActivity.mockResolvedValue({ + id: 'comment-3d-1', + creatorId: 'user-c1', + message: 'Look at the back', + replyToId: null, + second: 2.0, // 2.0s = 180° + attachments: [], + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'chat', + status: 'pending', + assetId: 'a-3d', + payload: { + projectId: 'p1', + agent: { + agentId: 'b1', + userCommentId: 'comment-3d-1', + }, + }, + }, + }) + + await agentChat(task) + + expect(mockActivities.agentChatActivity).toHaveBeenCalledWith( + expect.objectContaining({ + messageContext: expect.objectContaining({ + position: { type: 'degree', degree: 180 }, + }), + }), + ) + }) }) diff --git a/packages/agent/src/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts index 40ea7756..5e205273 100644 --- a/packages/agent/src/workflows/agent-chat.ts +++ b/packages/agent/src/workflows/agent-chat.ts @@ -8,6 +8,7 @@ import type { ShumaiMediaPosition, } from '@shumai/dtos' import { getProxyType } from '@shumai/core/src/utils/mime' +import { secondToDegree } from '@shumai/core/src/utils/model3d' export async function agentChat(task: WorkflowTask): Promise { const { @@ -273,7 +274,9 @@ export async function agentChat(task: WorkflowTask): Promise { // Build position let mediaPosition: ShumaiMediaPosition | undefined if (commentTimestamp !== undefined && commentTimestamp !== null) { - if (proxyType === 'pdf') { + if (proxyType === '3d') { + mediaPosition = { type: 'degree', degree: secondToDegree(commentTimestamp) } + } else if (proxyType === 'pdf') { mediaPosition = { type: 'page', page: Math.round(commentTimestamp) } } else { mediaPosition = { type: 'time', seconds: commentTimestamp } diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts index 2d0b2a84..54bae9d4 100644 --- a/packages/agent/src/workflows/agent-embedding.test.ts +++ b/packages/agent/src/workflows/agent-embedding.test.ts @@ -211,6 +211,68 @@ describe('Agent Embedding Workflow', () => { }) }) + it('should run agent embedding workflow successfully for 3D asset using 4-views image', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'model-1', + mediaType: 'model/gltf-binary', + storageKey: { key: 'raw/model.glb' }, + media: { + proxyType: '3d', + imageTranscodes: [{ key: 'files/views.webp', format: 'webp' }], + }, + }, + chunkDuration: 60.0, + chunkOverlap: 5.0, + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'model-1', + teamId: 't1', + }, + }) + + await agentEmbeddingMedia(task) + + expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({ + teamId: 't1', + assetKey: 'files/views.webp', + mediaType: 'image/webp', + }) + }) + + it('should reject 3D asset for embeddings if imageTranscodes is missing', async () => { + mockActivities.getEmbeddingContextActivity.mockResolvedValue({ + agent: { id: 'b1' }, + asset: { + id: 'model-1', + mediaType: 'model/gltf-binary', + storageKey: { key: 'raw/model.glb' }, + media: { + proxyType: '3d', + imageTranscodes: [], + }, + }, + }) + + const task = await prisma.workflowTask.create({ + data: { + type: 'ai_embedding', + status: 'pending', + assetId: 'model-1', + teamId: 't1', + }, + }) + + await expect(agentEmbeddingMedia(task)).rejects.toThrow( + 'unsupported media type for embeddings: model/gltf-binary', + ) + }) + it('should run agent embedding workflow successfully for video in chunks', async () => { mockActivities.getEmbeddingContextActivity.mockResolvedValue({ agent: { id: 'b1' }, diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index ab365ab7..23cb747d 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -70,7 +70,10 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // eslint-disable-next-line @typescript-eslint/no-explicit-any const { asset, chunkDuration, chunkOverlap } = context as any - const isImage = asset.mediaType?.startsWith('image/') + const mediaInfo = asset.media as PrismaJson.MediaInfo | null + const is3dWithViews = + mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0 + const isImage = asset.mediaType?.startsWith('image/') || is3dWithViews const isVideo = asset.mediaType?.startsWith('video/') if (!isImage && !isVideo) { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7c162257..76373a87 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -32,4 +32,5 @@ export * from './media-generation/media-generation' export * from './sandbox' export * from './asset/comments-export' export * from './turntable/turntable' +export * from './utils/model3d' export * from './integrations/helper' diff --git a/packages/core/src/transcode/transcode.test.ts b/packages/core/src/transcode/transcode.test.ts index 802c5086..ed9fb94d 100644 --- a/packages/core/src/transcode/transcode.test.ts +++ b/packages/core/src/transcode/transcode.test.ts @@ -4432,6 +4432,89 @@ describe('TranscodeService', () => { }) }) + describe('generate3dViews', () => { + it('executes ffmpeg with 2x2 grid filter, libwebp encoder, and q=85', async () => { + let executedArgs: string[] = [] + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(child_process.execFile as any).mockImplementation( + ( + _cmd: string, + args: string[], + cb: (err: unknown, result: { stdout: string; stderr: string }) => void, + ) => { + executedArgs = args + cb(null, { stdout: '', stderr: '' }) + }, + ) + + const inputPath = path.join(tempDir, 'turntable.mp4') + const outputPath = path.join(tempDir, 'views.webp') + + await transcodeService.generate3dViews(inputPath, outputPath) + + expect(child_process.execFile).toHaveBeenCalledWith( + 'ffmpeg', + expect.any(Array), + expect.any(Function), + ) + expect(executedArgs).toEqual([ + '-y', + '-loglevel', + 'warning', + '-i', + inputPath, + '-filter_complex', + "[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]", + '-map', + '[views_out]', + '-frames:v', + '1', + '-c:v', + 'libwebp', + '-q:v', + '85', + outputPath, + ]) + }) + + it('passes abort signal to execFile when provided', async () => { + const controller = new AbortController() + let optionsPassed: unknown = null + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(child_process.execFile as any).mockImplementation( + (_cmd: string, _args: string[], optionsOrCb: unknown, maybeCb?: unknown) => { + const cb = (typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb) as ( + err: unknown, + result: { stdout: string; stderr: string }, + ) => void + if (typeof optionsOrCb !== 'function') { + optionsPassed = optionsOrCb + } + cb(null, { stdout: '', stderr: '' }) + }, + ) + + const inputPath = path.join(tempDir, 'turntable.mp4') + const outputPath = path.join(tempDir, 'views.webp') + + await transcodeService.generate3dViews(inputPath, outputPath, controller.signal) + + expect(optionsPassed).toMatchObject({ signal: controller.signal }) + }) + + it('rejects with cancellation error if aborted before execution', async () => { + const controller = new AbortController() + controller.abort() + + const inputPath = path.join(tempDir, 'aborted.mp4') + const outputPath = path.join(tempDir, 'views.webp') + + await expect( + transcodeService.generate3dViews(inputPath, outputPath, controller.signal), + ).rejects.toThrow('3D views generation cancelled') + }) + }) + describe('HLS utilities', () => { it('builds master playlist correctly with bandwidth and codecs', () => { const playlist = buildHlsMasterPlaylist([ diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts index a80eb924..ccf197d7 100644 --- a/packages/core/src/transcode/transcode.ts +++ b/packages/core/src/transcode/transcode.ts @@ -3192,6 +3192,37 @@ export class TranscodeService { await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args]) } } + + async generate3dViews( + inputFile: string, + outputViews: string, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) { + throw new Error('3D views generation cancelled') + } + const filterComplex = `[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]` + const args = [ + '-i', + inputFile, + '-filter_complex', + filterComplex, + '-map', + '[views_out]', + '-frames:v', + '1', + '-c:v', + 'libwebp', + '-q:v', + '85', + outputViews, + ] + if (signal) { + await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal }) + } else { + await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args]) + } + } } function renderAnnotationsToSvg( diff --git a/packages/core/src/utils/model3d.test.ts b/packages/core/src/utils/model3d.test.ts new file mode 100644 index 00000000..ff49cf6e --- /dev/null +++ b/packages/core/src/utils/model3d.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' +import { + calculateTurntableFrameCenterTime, + degreeToSecond, + degreeToTurntableFrame, + isValidTurntableDegree, + secondToDegree, + secondToTurntableFrame, + TOTAL_TURNTABLE_FRAMES, + TURNTABLE_FPS, +} from './model3d' + +describe('model3d utils', () => { + describe('secondToTurntableFrame', () => { + it('converts playback seconds to frame indices (0..23)', () => { + expect(secondToTurntableFrame(0)).toBe(0) + expect(secondToTurntableFrame(0.167)).toBe(1) + expect(secondToTurntableFrame(0.5)).toBe(3) + expect(secondToTurntableFrame(2.0)).toBe(12) + expect(secondToTurntableFrame(3.9)).toBe(23) + }) + + it('clamps out-of-range seconds', () => { + expect(secondToTurntableFrame(-1)).toBe(0) + expect(secondToTurntableFrame(10)).toBe(TOTAL_TURNTABLE_FRAMES - 1) + }) + }) + + describe('secondToDegree', () => { + it('converts seconds to turntable degrees (0, 15, ..., 345)', () => { + expect(secondToDegree(0)).toBe(0) + expect(secondToDegree(1 / TURNTABLE_FPS)).toBe(15) + expect(secondToDegree(0.5)).toBe(45) + expect(secondToDegree(2.0)).toBe(180) + expect(secondToDegree(3.9)).toBe(345) + }) + }) + + describe('degreeToTurntableFrame', () => { + it('converts discrete degrees to frame indices', () => { + expect(degreeToTurntableFrame(0)).toBe(0) + expect(degreeToTurntableFrame(15)).toBe(1) + expect(degreeToTurntableFrame(45)).toBe(3) + expect(degreeToTurntableFrame(90)).toBe(6) + expect(degreeToTurntableFrame(180)).toBe(12) + expect(degreeToTurntableFrame(345)).toBe(23) + }) + + it('normalizes degrees outside [0, 360)', () => { + expect(degreeToTurntableFrame(360)).toBe(0) + expect(degreeToTurntableFrame(-15)).toBe(23) + expect(degreeToTurntableFrame(720)).toBe(0) + }) + }) + + describe('calculateTurntableFrameCenterTime and degreeToSecond', () => { + it('calculates frame center times within bounds', () => { + const centerTime0 = calculateTurntableFrameCenterTime(0) + expect(centerTime0).toBeCloseTo(0.5 / TURNTABLE_FPS) + + const centerTime23 = calculateTurntableFrameCenterTime(23) + expect(centerTime23).toBeLessThan(TOTAL_TURNTABLE_FRAMES / TURNTABLE_FPS) + expect(centerTime23).toBeGreaterThan(23 / TURNTABLE_FPS) + + expect(degreeToSecond(0)).toBeCloseTo(centerTime0) + expect(degreeToSecond(45)).toBeCloseTo((3 + 0.5) / TURNTABLE_FPS) + expect(degreeToSecond(345)).toBeCloseTo(centerTime23) + }) + }) + + describe('isValidTurntableDegree', () => { + it('returns true for integer multiples of 15 from 0 to 345', () => { + for (let deg = 0; deg < 360; deg += 15) { + expect(isValidTurntableDegree(deg)).toBe(true) + } + }) + + it('returns false for non-multiples of 15', () => { + expect(isValidTurntableDegree(10)).toBe(false) + expect(isValidTurntableDegree(44)).toBe(false) + expect(isValidTurntableDegree(45.5)).toBe(false) + }) + + it('returns false for out of range or negative degrees', () => { + expect(isValidTurntableDegree(-15)).toBe(false) + expect(isValidTurntableDegree(360)).toBe(false) + expect(isValidTurntableDegree(720)).toBe(false) + }) + }) +}) diff --git a/packages/core/src/utils/model3d.ts b/packages/core/src/utils/model3d.ts new file mode 100644 index 00000000..0c32cf4b --- /dev/null +++ b/packages/core/src/utils/model3d.ts @@ -0,0 +1,58 @@ +export const TOTAL_TURNTABLE_FRAMES = 24 +export const TURNTABLE_FPS = 6 +export const TURNTABLE_FRAME_DURATION = 1 / TURNTABLE_FPS + +/** + * Converts a playback second into a turntable frame index (0..23), clamped. + */ +export function secondToTurntableFrame(second: number): number { + return Math.min( + Math.max(0, Math.floor(second * TURNTABLE_FPS + 1e-4)), + TOTAL_TURNTABLE_FRAMES - 1, + ) +} + +/** + * Converts a turntable playback second to turntable degree (0, 15, 30, ..., 345). + */ +export function secondToDegree(second: number): number { + const frameIndex = secondToTurntableFrame(second) + return (frameIndex * 15) % 360 +} + +/** + * Calculates the presentation time within a turntable frame. + * Seeking near frame center avoids PTS boundary inaccuracies in video decoders. + */ +export function calculateTurntableFrameCenterTime(frameIndex: number): number { + const normalizedFrame = + ((frameIndex % TOTAL_TURNTABLE_FRAMES) + TOTAL_TURNTABLE_FRAMES) % TOTAL_TURNTABLE_FRAMES + if (normalizedFrame === TOTAL_TURNTABLE_FRAMES - 1) { + return normalizedFrame * TURNTABLE_FRAME_DURATION + TURNTABLE_FRAME_DURATION * 0.25 + } + return (normalizedFrame + 0.5) * TURNTABLE_FRAME_DURATION +} + +/** + * Converts a rotation degree (0..345) to a turntable frame index (0..23). + */ +export function degreeToTurntableFrame(degree: number): number { + const normalizedDegree = ((Math.round(degree) % 360) + 360) % 360 + return Math.round(normalizedDegree / 15) % TOTAL_TURNTABLE_FRAMES +} + +/** + * Converts a rotation degree into a turntable video playback second (center time of that frame). + */ +export function degreeToSecond(degree: number): number { + const frameIndex = degreeToTurntableFrame(degree) + return calculateTurntableFrameCenterTime(frameIndex) +} + +/** + * Validates whether a degree is a valid discrete turntable angle: + * integer multiple of 15 between 0 and 345 inclusive (0, 15, 30, ..., 345). + */ +export function isValidTurntableDegree(degree: number): boolean { + return Number.isInteger(degree) && degree >= 0 && degree <= 345 && degree % 15 === 0 +} diff --git a/packages/dtos/src/chat.ts b/packages/dtos/src/chat.ts index 1159ba5a..2705dd16 100644 --- a/packages/dtos/src/chat.ts +++ b/packages/dtos/src/chat.ts @@ -47,7 +47,7 @@ export const shumaiAssetContextSchema = z.object({ id: z.string(), name: z.string(), type: z.string(), - mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(), + mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(), mimeType: z.string().optional(), parentId: z.string().optional(), path: z.string().optional(), @@ -64,7 +64,7 @@ export const shumaiAttachedFileContextSchema = z.object({ id: z.string(), name: z.string(), type: z.string(), - mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(), + mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(), mimeType: z.string().optional(), path: z.string().optional(), url: z.string().optional(), @@ -75,6 +75,7 @@ export type ShumaiAttachedFileContext = z.infer diff --git a/packages/dtos/src/context.ts b/packages/dtos/src/context.ts index 2570f0e9..e8192e10 100644 --- a/packages/dtos/src/context.ts +++ b/packages/dtos/src/context.ts @@ -104,6 +104,8 @@ export function serializeContextToXml(context?: ShumaiMessageContext | null): st lines.push(` `) } else if (context.position.type === 'page') { lines.push(` `) + } else if (context.position.type === 'degree') { + lines.push(` `) } } diff --git a/packages/transcode/src/activities/transcode-3d.test.ts b/packages/transcode/src/activities/transcode-3d.test.ts index 72c2a34f..ad22225b 100644 --- a/packages/transcode/src/activities/transcode-3d.test.ts +++ b/packages/transcode/src/activities/transcode-3d.test.ts @@ -4,6 +4,7 @@ import { start3dRenderActivity, waitFor3dVideoRenderActivity, generate3dSpriteActivity, + generate3dViewsActivity, } from './transcode' import { turntableService } from '@shumai/core/src/turntable/turntable' import { s3Service } from '@shumai/core/src/s3/s3' @@ -31,6 +32,7 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({ transcodeService: { convertImageToWebp: vi.fn(), generate3dSprite: vi.fn(), + generate3dViews: vi.fn(), }, })) @@ -55,6 +57,7 @@ describe('3D Transcode Activities', () => { vi.mocked(fs.readFileSync).mockReturnValue(Buffer.from('mock-bytes')) vi.mocked(transcodeService.convertImageToWebp).mockResolvedValue(Buffer.from('webp-bytes')) vi.mocked(transcodeService.generate3dSprite).mockResolvedValue() + vi.mocked(transcodeService.generate3dViews).mockResolvedValue() vi.mocked(s3Service.putObject).mockResolvedValue({} as never) vi.mocked(turntableService.deleteTask).mockResolvedValue(true) }) @@ -324,4 +327,29 @@ describe('3D Transcode Activities', () => { expect(res.spriteKey).toBe('files/asset-1/sprite.webp') }) }) + + describe('generate3dViewsActivity', () => { + it('generates 3d 4-views composite image and uploads to S3', async () => { + const res = await generate3dViewsActivity({ + taskId: 'wf-task-4', + assetKey: 'files/asset-1/model.glb', + videoFilePath: '/tmp/turntable.mp4', + viewsKey: 'files/asset-1/views.webp', + }) + + expect(transcodeService.generate3dViews).toHaveBeenCalledWith( + '/tmp/turntable.mp4', + '/tmp/views.webp', + undefined, + ) + expect(s3Service.putObject).toHaveBeenCalledWith( + 'shumai', + 'files/asset-1/views.webp', + expect.any(Buffer), + expect.any(Number), + 'image/webp', + ) + expect(res.viewsKey).toBe('files/asset-1/views.webp') + }) + }) }) diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 9a39c250..afd8fcf2 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -27,6 +27,7 @@ import { transcodeVideoChunkActivity, deleteS3ObjectActivity, createAutofillTaskIfEnabledActivity, + createEmbeddingTaskIfEnabledActivity, } from './transcode' vi.mock('@shumai/core/src/s3/s3', () => ({ @@ -1717,4 +1718,115 @@ describe('Transcode Activities', () => { expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id) }) }) + + describe('createEmbeddingTaskIfEnabledActivity', () => { + it('should create ai_embedding task for 3D asset with imageTranscodes', async () => { + const user = await prisma.user.create({ + data: { name: 'Test User Embedding', email: 'test-embedding-agent@test.com' }, + }) + const team = await prisma.team.create({ + data: { name: 'Test Team Embedding' }, + }) + await prisma.teamMember.create({ + data: { teamId: team.id, userId: user.id, role: 'owner' }, + }) + const project = await prisma.project.create({ + data: { name: 'Test Project Embedding', teamId: team.id }, + }) + const embeddingAgent = await prisma.agent.create({ + data: { + id: user.id, + type: 'embedding', + enabled: true, + teamId: team.id, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + config: { provider: 'test', model: 'test' } as any, + }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'robot.glb', + mediaType: 'model/gltf-binary', + type: 'file', + status: 'processed', + projectId: project.id, + media: { + proxyType: '3d', + imageTranscodes: [ + { + key: 'files/asset-1/views.webp', + width: 2880, + height: 2880, + quality: 85, + format: 'webp', + }, + ], + } as unknown as PrismaJson.MediaInfo, + }, + }) + + await createEmbeddingTaskIfEnabledActivity({ + assetId: asset.id, + teamId: team.id, + projectId: project.id, + }) + + const task = await prisma.workflowTask.findFirst({ + where: { assetId: asset.id, type: 'ai_embedding' }, + }) + expect(task).toBeDefined() + expect(task?.status).toBe('pending') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + expect((task?.payload as any)?.agent?.agentId).toBe(embeddingAgent.id) + }) + + it('should NOT create ai_embedding task for 3D asset without imageTranscodes', async () => { + const user = await prisma.user.create({ + data: { name: 'Test User Embedding 2', email: 'test-embedding-agent-2@test.com' }, + }) + const team = await prisma.team.create({ + data: { name: 'Test Team Embedding 2' }, + }) + await prisma.teamMember.create({ + data: { teamId: team.id, userId: user.id, role: 'owner' }, + }) + const project = await prisma.project.create({ + data: { name: 'Test Project Embedding 2', teamId: team.id }, + }) + await prisma.agent.create({ + data: { + id: user.id, + type: 'embedding', + enabled: true, + teamId: team.id, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + config: { provider: 'test', model: 'test' } as any, + }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'robot.glb', + mediaType: 'model/gltf-binary', + type: 'file', + status: 'processed', + projectId: project.id, + media: { + proxyType: '3d', + imageTranscodes: [], + } as unknown as PrismaJson.MediaInfo, + }, + }) + + await createEmbeddingTaskIfEnabledActivity({ + assetId: asset.id, + teamId: team.id, + projectId: project.id, + }) + + const task = await prisma.workflowTask.findFirst({ + where: { assetId: asset.id, type: 'ai_embedding' }, + }) + expect(task).toBeNull() + }) + }) }) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index ad67f90f..1f35d5fe 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -1360,8 +1360,11 @@ export async function createEmbeddingTaskIfEnabledActivity( return } + const mediaInfo = asset.media as PrismaJson.MediaInfo | null const isVideo = asset.mediaType?.startsWith('video/') - const isImage = asset.mediaType?.startsWith('image/') + const isImage = + asset.mediaType?.startsWith('image/') || + (mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0) if (!isVideo && !isImage) { return } @@ -1886,3 +1889,51 @@ export async function generate3dSpriteActivity( }) } } + +export interface Generate3dViewsActivityParams { + taskId: string + assetKey: string + videoFilePath: string + viewsKey: string +} + +export async function generate3dViewsActivity( + params: Generate3dViewsActivityParams, +): Promise<{ viewsKey: string }> { + const signal = getActivityCancellationSignal(params.taskId) + if (signal?.aborted) { + throw ApplicationFailure.create({ + message: '3D views generation cancelled', + nonRetryable: true, + }) + } + + const bucket = process.env.S3_BUCKET || 'shumai' + const tmpDir = path.dirname(params.videoFilePath) + const viewsFilePath = path.join(tmpDir, 'views.webp') + + try { + await transcodeService.generate3dViews(params.videoFilePath, viewsFilePath, signal) + const viewsBuffer = fs.readFileSync(viewsFilePath) + + await ensureAssetNotPurging(params.assetKey) + await s3Service.putObject( + bucket, + params.viewsKey, + viewsBuffer, + viewsBuffer.length, + 'image/webp', + ) + + return { viewsKey: params.viewsKey } + } catch (err) { + if (err instanceof ApplicationFailure) { + throw err + } + const errMsg = err instanceof Error ? err.message : String(err) + throw ApplicationFailure.create({ + message: `Failed to generate 3D views: ${errMsg}`, + nonRetryable: true, + }) + } +} diff --git a/packages/transcode/src/workflows/transcode-3d.test.ts b/packages/transcode/src/workflows/transcode-3d.test.ts index 494a08c3..cacbeadd 100644 --- a/packages/transcode/src/workflows/transcode-3d.test.ts +++ b/packages/transcode/src/workflows/transcode-3d.test.ts @@ -29,6 +29,7 @@ describe('transcode3dWorkflow', () => { _activityName: 'waitFor3dVideoRenderActivity', }), generate3dSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generate3dSpriteActivity' }), + generate3dViewsActivity: Object.assign(vi.fn(), { _activityName: 'generate3dViewsActivity' }), updateAssetMediaActivity: Object.assign(vi.fn(), { _activityName: 'updateAssetMediaActivity', }), @@ -173,6 +174,13 @@ describe('transcode3dWorkflow', () => { spriteKey: 'files/asset-3d/sprite.webp', }) + expect(mockActivities.generate3dViewsActivity).toHaveBeenCalledWith({ + taskId: 'task-3d', + assetKey: 'files/asset-3d/robot.glb', + videoFilePath: '/tmp/turntable.mp4', + viewsKey: 'files/asset-3d/views.webp', + }) + expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith( expect.objectContaining({ assetId: 'asset-3d', @@ -182,6 +190,15 @@ describe('transcode3dWorkflow', () => { frames: 24, poster: { key: 'files/asset-3d/poster.webp' }, sprite: expect.objectContaining({ key: 'files/asset-3d/sprite.webp' }), + imageTranscodes: [ + { + key: 'files/asset-3d/views.webp', + width: 2880, + height: 2880, + quality: 85, + format: 'webp', + }, + ], }), }), ) diff --git a/packages/transcode/src/workflows/transcode-3d.ts b/packages/transcode/src/workflows/transcode-3d.ts index 24597f9f..496b8d02 100644 --- a/packages/transcode/src/workflows/transcode-3d.ts +++ b/packages/transcode/src/workflows/transcode-3d.ts @@ -23,6 +23,7 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise { start3dRenderActivity, waitFor3dVideoRenderActivity, generate3dSpriteActivity, + generate3dViewsActivity, updateAssetMediaActivity, createEmbeddingTaskIfEnabledActivity, createAutofillTaskIfEnabledActivity, @@ -46,6 +47,7 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise { const videoKey = assetDir ? `${assetDir}/turntable.mp4` : 'turntable.mp4' const posterKey = assetDir ? `${assetDir}/poster.webp` : 'poster.webp' const spriteKey = assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp' + const viewsKey = assetDir ? `${assetDir}/views.webp` : 'views.webp' const renderStart = await executeActivity(workerQueue, start3dRenderActivity, { taskId: task.id, @@ -117,6 +119,23 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise { tileY: 10, } + await executeActivity(workerQueue, generate3dViewsActivity, { + taskId: task.id, + assetKey: key, + videoFilePath: renderVideo.videoFilePath, + viewsKey, + }) + + mediaInfo.imageTranscodes = [ + { + key: viewsKey, + width: 2880, + height: 2880, + quality: 85, + format: 'webp', + }, + ] + await executeActivity(workerQueue, updateAssetMediaActivity, { assetId: asset.id, mediaInfo,