diff --git a/CHANGELOG.md b/CHANGELOG.md
index 09a8d74a..3d6fdd0c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Fixed
+* **agent**: Fix an issue where 3D visual markup annotations in 1-on-1 chat sessions were inspected at 0° instead of the user's annotated degree angle
* **transcode**: Fix an issue where portrait photos from cameras and phones appeared sideways in thumbnails, previews, watermarked share links and annotated snapshots, and were listed with their width and height swapped
### Changed
diff --git a/packages/agent/src/activities/agent.test.ts b/packages/agent/src/activities/agent.test.ts
index 1980151b..3575f69e 100644
--- a/packages/agent/src/activities/agent.test.ts
+++ b/packages/agent/src/activities/agent.test.ts
@@ -662,6 +662,23 @@ describe('Agent Activities', () => {
const pdfPrompt = mockHarness.prompt.mock.calls[1]?.[0] ?? ''
expect(pdfPrompt).toContain('Document pages: 50')
expect(pdfPrompt).not.toContain('Video duration')
+
+ // For a 3D asset, should produce 3D Model recommendation
+ await autofillAiActivity({
+ teamId: 't1',
+ assetId: 'model-3',
+ assetName: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ images: [],
+ fields: [{ id: 'f1', config: { name: 'F1', type: 'text' } }],
+ context,
+ })
+
+ const modelPrompt = mockHarness.prompt.mock.calls[2]?.[0] ?? ''
+ expect(modelPrompt).toContain('3D Model: This asset has an orthogonal 4-view composite image')
+ expect(modelPrompt).toContain(
+ 'Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right',
+ )
})
})
@@ -1170,6 +1187,43 @@ describe('Agent Database Activities Integration', () => {
expect((rootEntry as any)?.details?.thread).toBeUndefined()
})
+ it('should set media position to degree for 3D asset comments', async () => {
+ const asset3d = await prisma.asset.create({
+ data: {
+ name: 'robot.glb',
+ type: 'file',
+ status: 'processed',
+ mediaType: 'model/gltf-binary',
+ media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo,
+ projectId: project.id,
+ },
+ })
+
+ const comment3d = await prisma.assetComment.create({
+ data: {
+ assetId: asset3d.id,
+ message: 'Nice angle',
+ creatorId: user.id,
+ second: 0.5, // 45°
+ },
+ })
+
+ const sessionId = await initializeAgentSessionActivity({
+ teamId: team.id,
+ agentId: 'test-agent-id',
+ userCommentId: comment3d.id,
+ userId: user.id,
+ })
+
+ const storage = new DatabaseSessionStorage(sessionId)
+ const pathEntries = await storage.getPathToRoot(comment3d.id)
+ const entry = pathEntries.find((e) => e.id === comment3d.id)
+
+ expect(entry).toBeDefined()
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- test entry position parsing
+ expect((entry as any)?.details?.position).toEqual({ type: 'degree', degree: 45 })
+ })
+
it('should return user name and role for team member', async () => {
const info = await getUserTeamInfoActivity({
userId: user.id,
@@ -2105,5 +2159,83 @@ describe('Agent Database Activities Integration', () => {
expect(systemPromptArg).toContain('')
expect(systemPromptArg).toContain('read_thread')
})
+
+ it('agentChatActivity injects # 3D Model Inspection into systemPrompt', async () => {
+ const user = await prisma.user.create({
+ data: { name: 'User_3d_Prompt', email: `prompt-3d-${Date.now()}@example.com` },
+ })
+ const team = await prisma.team.create({
+ data: { name: 'Team_3d_Prompt_' + Date.now() },
+ })
+ const project = await prisma.project.create({
+ data: { name: 'Project_3d_Prompt', teamId: team.id },
+ })
+ const asset = await prisma.asset.create({
+ data: {
+ name: 'robot.glb',
+ type: 'file',
+ status: 'processed',
+ mediaType: 'model/gltf-binary',
+ media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo,
+ projectId: project.id,
+ },
+ })
+
+ const comment = await prisma.assetComment.create({
+ data: {
+ assetId: asset.id,
+ creatorId: user.id,
+ message: 'Check this 3d model',
+ },
+ })
+
+ const mockSession = {
+ id: 'mock-session-id',
+ subscribe: vi.fn(),
+ getEntries: vi.fn().mockResolvedValue([]),
+ getStorage: vi.fn().mockReturnValue({ sessionId: 'mock-session-id' }),
+ }
+ const mockHarness = {
+ subscribe: vi.fn(),
+ prompt: vi.fn().mockResolvedValue({
+ content: [{ type: 'text', text: 'Response' }],
+ usage: { input: 10, output: 20 },
+ }),
+ }
+
+ vi.mocked(piAgent.createAgentSession).mockResolvedValue({
+ session: mockSession as unknown as Session,
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock AgentHarness instance for activity test
+ harness: mockHarness as unknown as AgentHarness,
+ })
+
+ const context = {
+ agent: { id: 'agent-1', provider: { name: 'google' }, modelRef: { modelId: 'gemini' } },
+ dbProviders: [],
+ teamSkills: [],
+ allowedDomains: [],
+ } as unknown as AgentExecutionContext
+
+ await agentChatActivity({
+ teamId: team.id,
+ agentId: 'agent-1',
+ message: 'Hello',
+ imageUrls: [],
+ projectId: project.id,
+ folderId: asset.id,
+ assetId: asset.id,
+ sessionId: 'mock-session-id',
+ userCommentId: comment.id,
+ userId: user.id,
+ context,
+ })
+
+ const lastCall = vi.mocked(piAgent.createAgentSession).mock.calls.at(-1)
+ const systemPromptArg = lastCall?.[0].systemPrompt as string
+ expect(systemPromptArg).toContain('# 3D Model Inspection')
+ expect(systemPromptArg).toContain('media_type="3d"')
+ expect(systemPromptArg).toContain('config3d')
+ expect(systemPromptArg).toContain('')
+ })
})
})
diff --git a/packages/agent/src/activities/agent.ts b/packages/agent/src/activities/agent.ts
index 6942d726..bf03c10f 100644
--- a/packages/agent/src/activities/agent.ts
+++ b/packages/agent/src/activities/agent.ts
@@ -21,6 +21,7 @@ import {
import { aiUsageService } from '@shumai/core/src/ai-usage/ai-usage'
import { quotaService, QuotaExceededError } from '@shumai/core/src/quota/quota-service'
import { getProxyType } from '@shumai/core/src/utils/mime'
+import { secondToDegree } from '@shumai/core/src/utils/model3d'
import {
UpdateAssetMetadataRequest,
type ShumaiMessageContext,
@@ -179,6 +180,8 @@ User messages may contain a block detailing the user, active asset loc
systemPrompt += `\n\n# Comment Threads\nWhen operating in comment mode, previous top-level messages in the conversation history may contain a tag indicating an earlier discussion thread with replies. If a user's question refers to or depends on earlier comments or discussions, you can use the 'read_thread' tool with the thread ID to inspect all replies in that thread.`
}
+ systemPrompt += `\n\n# 3D Model Inspection\nWhen interacting with 3D model assets (media_type="3d"), you can visually inspect the asset using 'read_asset'. Calling 'read_asset' with config3d: null (or config3d: {}) returns an orthogonal 4-view composite image (Front at top-left, Right side at top-right, Back at bottom-left, Left side at bottom-right). If a tag is present or a specific angle is discussed, pass 'config3d: { degree: ... }' (integer 0, 15, 30, ..., 345) to inspect that view angle. If an tag is present on a 3D asset, pass 'annotationId' to inspect the angle with the user's visual markup drawings overlaid.`
+
if (agent.type === 'autofill') {
systemPrompt += `\n\n# Autofill Instructions\nWhen performing metadata autofill, inspect the asset content using the 'read_asset' tool. If 'read_asset' fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.`
}
@@ -564,6 +567,7 @@ export async function autofillAiActivity(params: AutofillAiParams) {
const proxyType = getProxyType(params.mediaType, params.assetName)
const isVideoOrAudio = proxyType === 'video' || proxyType === 'audio'
const isPdf = proxyType === 'pdf'
+ const is3d = proxyType === '3d'
if (params.duration !== undefined && params.duration > 0 && (!proxyType || isVideoOrAudio)) {
promptLines.push(
@@ -575,11 +579,16 @@ export async function autofillAiActivity(params: AutofillAiParams) {
`- Document pages: ${params.pageCount}. Recommended inspection: call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ${Math.min(params.pageCount, 20)} } or mode: "text".`,
)
}
+ if (is3d) {
+ promptLines.push(
+ '- 3D Model: This asset has an orthogonal 4-view composite image (2x2 grid showing Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right). Call read_asset to inspect the 4 views.',
+ )
+ }
promptLines.push(
'',
'Instructions:',
- `1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type).`,
+ `1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type; for 3D assets, call read_asset directly to inspect the 4-view composite image).`,
'2. After analyzing the retrieved content, call the "autofill_metadata" tool to provide the extracted metadata values.',
'3. If "read_asset" fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.',
)
@@ -834,7 +843,9 @@ export async function initializeAgentSessionActivity(params: {
let position: ShumaiMediaPosition | undefined = undefined
if (c.second !== null && c.second !== undefined) {
- if (proxyType === 'pdf') {
+ if (proxyType === '3d') {
+ position = { type: 'degree', degree: secondToDegree(c.second) }
+ } else if (proxyType === 'pdf') {
position = { type: 'page', page: Math.round(c.second) }
} else {
position = { type: 'time', seconds: c.second }
@@ -852,7 +863,14 @@ export async function initializeAgentSessionActivity(params: {
id: att.asset.id,
name: att.asset.name,
type: att.asset.type,
- mediaType: attProxyType as 'image' | 'video' | 'pdf' | 'audio' | 'other' | undefined,
+ mediaType: attProxyType as
+ | 'image'
+ | 'video'
+ | 'pdf'
+ | 'audio'
+ | '3d'
+ | 'other'
+ | undefined,
mimeType: att.asset.mediaType || undefined,
})
}
diff --git a/packages/agent/src/context/serialize-context.test.ts b/packages/agent/src/context/serialize-context.test.ts
index 1c72c542..1811fcf6 100644
--- a/packages/agent/src/context/serialize-context.test.ts
+++ b/packages/agent/src/context/serialize-context.test.ts
@@ -350,5 +350,24 @@ describe('serializeContextToXml', () => {
'\n \n',
)
})
+
+ it('serializes position type="degree" with integer degree attribute without seconds', () => {
+ const context: ShumaiMessageContext = {
+ currentAsset: {
+ id: 'ast_3d',
+ name: 'robot.glb',
+ type: 'file',
+ mediaType: '3d',
+ path: 'robot.glb',
+ },
+ position: {
+ type: 'degree',
+ degree: 45,
+ },
+ }
+ expect(serializeContextToXml(context)).toBe(
+ '\n \n \n',
+ )
+ })
})
})
diff --git a/packages/agent/src/tools/annotation-resolver.test.ts b/packages/agent/src/tools/annotation-resolver.test.ts
index 2ea0e0df..5452944b 100644
--- a/packages/agent/src/tools/annotation-resolver.test.ts
+++ b/packages/agent/src/tools/annotation-resolver.test.ts
@@ -4,6 +4,7 @@ import { setupTestDbHooks } from '@shumai/db/test'
import { resolveAnnotationsById } from './annotation-resolver'
import { DatabaseSessionStorage } from '../database-session-storage'
import type { SessionTreeEntry } from '@earendil-works/pi-agent-core'
+import { degreeToSecond } from '@shumai/core/src/utils/model3d'
describe('resolveAnnotationsById', () => {
setupTestDbHooks()
@@ -210,6 +211,89 @@ describe('resolveAnnotationsById', () => {
])
})
+ it('resolves annotations and degree position timestamp from 1-on-1 chat AgentSessionEntry for 3D assets', async () => {
+ const user = await prisma.user.create({
+ data: { name: 'User 3D', email: 'user3d@test.com', password: 'pw' },
+ })
+ const team = await prisma.team.create({
+ data: { name: 'Team 3D' },
+ })
+ const project = await prisma.project.create({
+ data: { name: 'Project 3D', teamId: team.id },
+ })
+ const asset = await prisma.asset.create({
+ data: {
+ name: 'robot.glb',
+ type: 'file',
+ projectId: project.id,
+ status: 'uploaded',
+ },
+ })
+ const agentUser = await prisma.user.create({
+ data: { name: 'Agent 3D', email: 'agent3d@example.com', type: 'agent' },
+ })
+ const agent = await prisma.agent.create({
+ data: {
+ id: agentUser.id,
+ teamId: team.id,
+ type: 'chat',
+ config: { provider: 'test-provider', model: 'test-model' },
+ },
+ })
+
+ const storage = await DatabaseSessionStorage.create({
+ agentId: agent.id,
+ userId: user.id,
+ assetId: asset.id,
+ })
+
+ storage.currentMessageContext = {
+ currentAsset: {
+ id: asset.id,
+ name: asset.name,
+ type: 'file',
+ path: asset.name,
+ },
+ position: { type: 'degree', degree: 45 },
+ annotation: true,
+ annotations: [
+ {
+ type: 'rectangle',
+ x: 10,
+ y: 20,
+ width: 30,
+ height: 40,
+ },
+ ],
+ }
+
+ const userEntry: SessionTreeEntry = {
+ type: 'message',
+ id: '01JTEST3DENTRY123456789',
+ parentId: null,
+ timestamp: new Date().toISOString(),
+ message: {
+ role: 'user',
+ content: 'Check markup at 45 degrees',
+ timestamp: Date.now(),
+ },
+ }
+
+ await storage.appendEntry(userEntry)
+
+ const result = await resolveAnnotationsById(asset.id, '01JTEST3DENTRY123456789')
+ expect(result.timestamp).toBeCloseTo(degreeToSecond(45))
+ expect(result.annotations).toEqual([
+ {
+ type: 'rectangle',
+ x: 10,
+ y: 20,
+ width: 30,
+ height: 40,
+ },
+ ])
+ })
+
it('throws an error when AgentSessionEntry belongs to a different asset', async () => {
const user = await prisma.user.create({
data: { name: 'User', email: 'user3@test.com', password: 'pw' },
diff --git a/packages/agent/src/tools/annotation-resolver.ts b/packages/agent/src/tools/annotation-resolver.ts
index 97128e1f..0f928614 100644
--- a/packages/agent/src/tools/annotation-resolver.ts
+++ b/packages/agent/src/tools/annotation-resolver.ts
@@ -1,5 +1,6 @@
import { prisma } from '@shumai/db'
import type { ShumaiMessageContext } from '@shumai/dtos'
+import { degreeToSecond } from '@shumai/core/src/utils/model3d'
export async function resolveAnnotationsById(
assetId: string,
@@ -44,7 +45,14 @@ export async function resolveAnnotationsById(
const list = details?.annotations
const annotations =
Array.isArray(list) && list.length > 0 ? (list as unknown as PrismaJson.AnnotationList) : null
- const timestamp = details?.position?.type === 'time' ? details.position.seconds : null
+ let timestamp: number | null = null
+ if (details?.position?.type === 'time') {
+ timestamp = details.position.seconds
+ } else if (details?.position?.type === 'degree') {
+ timestamp = degreeToSecond(details.position.degree)
+ } else if (details?.position?.type === 'page') {
+ timestamp = details.position.page
+ }
return { annotations, timestamp }
}
diff --git a/packages/agent/src/tools/read-asset.test.ts b/packages/agent/src/tools/read-asset.test.ts
index a9b86a68..1073e640 100644
--- a/packages/agent/src/tools/read-asset.test.ts
+++ b/packages/agent/src/tools/read-asset.test.ts
@@ -11,6 +11,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
import * as annotationResolver from './annotation-resolver'
+import { degreeToSecond } from '@shumai/core/src/utils/model3d'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal()
@@ -211,6 +212,476 @@ describe('readAssetTool', () => {
})
})
+ describe('3D Assets', () => {
+ it('should read 3D asset 4-views image and return image with quadrant layout description', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ storageKey: { key: 'raw/robot.glb' },
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [{ key: 'proxy/views.webp' }],
+ },
+ } as unknown as Asset)
+
+ vi.mocked(s3Service.getObject).mockResolvedValue({
+ buffer: Buffer.from('fake-views-bytes'),
+ contentType: 'image/webp',
+ } as unknown as { buffer: Buffer; contentType: string })
+
+ const tool = createReadAssetTool('user-1')
+ const result = await tool.execute('call-3d', {
+ assetId: 'model-1',
+ annotationId: null,
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ })
+
+ expect(authzService.hasPermission).toHaveBeenCalledWith({
+ user: { id: 'user-1' },
+ permission: 'Read',
+ type: 'asset',
+ id: 'model-1',
+ })
+ expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/views.webp')
+
+ const textItem = result.content[0] as { type: 'text'; text: string }
+ const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
+
+ expect(textItem.text).toContain('Orthogonal 4-view composite image')
+ expect(textItem.text).toContain('Top-Left quadrant: Front view (0°)')
+ expect(textItem.text).toContain('Top-Right quadrant: Right side view (90°)')
+ expect(textItem.text).toContain('Bottom-Left quadrant: Back view (180°)')
+ expect(textItem.text).toContain('Bottom-Right quadrant: Left side view (270°)')
+ expect(imageItem.type).toBe('image')
+ expect(imageItem.data).toBe(Buffer.from('fake-views-bytes').toString('base64'))
+ expect(result.details.key).toBe('proxy/views.webp')
+ })
+
+ it('should support s3KeyOnly mode for 3D assets', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [{ key: 'proxy/views.webp' }],
+ },
+ } as unknown as Asset)
+
+ const tool = createReadAssetTool('user-1')
+ const result = await tool.execute('call-3d-s3', {
+ assetId: 'model-1',
+ annotationId: null,
+ s3KeyOnly: true,
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ })
+
+ expect(result.content).toHaveLength(1)
+ const textItem = result.content[0] as { type: 'text'; text: string }
+ expect(textItem.text).toContain('proxy/views.webp')
+ expect(textItem.text).toContain('Front view (0°)')
+ expect(result.details.key).toBe('proxy/views.webp')
+ })
+
+ it('should throw error if videoConfig is passed for 3D assets', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [{ key: 'proxy/views.webp' }],
+ },
+ } as unknown as Asset)
+
+ const tool = createReadAssetTool('user-1')
+ await expect(
+ tool.execute('call-3d-err', {
+ assetId: 'model-1',
+ annotationId: null,
+ imageConfig: null,
+ videoConfig: { start: 0, end: 4, count: 4 },
+ docConfig: null,
+ }),
+ ).rejects.toThrow(
+ "To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig",
+ )
+ })
+
+ it('should throw error if 3D asset imageTranscodes is missing when overview is requested', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [],
+ },
+ } as unknown as Asset)
+
+ const tool = createReadAssetTool('user-1')
+ await expect(
+ tool.execute('call-3d-err2', {
+ assetId: 'model-1',
+ annotationId: null,
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ }),
+ ).rejects.toThrow('4-views image (views.webp) is missing')
+ })
+
+ it('should extract specific turntable frame when config3d.degree is provided', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ projectId: 'proj-1',
+ media: {
+ proxyType: '3d',
+ videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
+ imageTranscodes: [{ key: 'proxy/views.webp' }],
+ },
+ } as unknown as Asset)
+
+ vi.mocked(prisma.workflowTask.create).mockResolvedValue({
+ id: 'task-shot-3d',
+ } as unknown as WorkflowTask)
+
+ vi.mocked(workflowService.executeWait).mockResolvedValue({
+ id: 'task-shot-3d',
+ status: WorkflowTaskStatus.completed,
+ output: {
+ screenshots: [{ key: 'proxy/shot-45.webp', timestamp: 0.5833 }],
+ },
+ } as unknown as WorkflowTask)
+
+ vi.mocked(s3Service.getObject).mockResolvedValue({
+ buffer: Buffer.from('fake-shot-45-bytes'),
+ contentType: 'image/webp',
+ } as unknown as { buffer: Buffer; contentType: string })
+
+ const tool = createReadAssetTool('user-1')
+ const result = await tool.execute('call-3d-degree', {
+ assetId: 'model-1',
+ annotationId: null,
+ config3d: { degree: 45 },
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ })
+
+ expect(prisma.workflowTask.create).toHaveBeenCalledWith({
+ data: expect.objectContaining({
+ type: WorkflowTaskType.transcode_screenshot,
+ assetId: 'model-1',
+ payload: {
+ projectId: 'proj-1',
+ screenshot: {
+ start: expect.any(Number),
+ end: expect.any(Number),
+ count: 1,
+ commentTimestamp: expect.any(Number),
+ annotations: undefined,
+ },
+ },
+ }),
+ })
+
+ const textItem = result.content[0] as { type: 'text'; text: string }
+ const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
+
+ expect(textItem.text).toContain('view at 45°:')
+ expect(imageItem.data).toBe(Buffer.from('fake-shot-45-bytes').toString('base64'))
+ expect(result.details.key).toBe('proxy/shot-45.webp')
+ })
+
+ it('should throw error when config3d.degree is not a multiple of 15 or out of range', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ media: {
+ proxyType: '3d',
+ videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
+ },
+ } as unknown as Asset)
+
+ const tool = createReadAssetTool('user-1')
+
+ await expect(
+ tool.execute('call-3d-invalid-deg', {
+ assetId: 'model-1',
+ annotationId: null,
+ config3d: { degree: 40 },
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ }),
+ ).rejects.toThrow('Invalid turntable degree: 40')
+
+ await expect(
+ tool.execute('call-3d-out-of-range', {
+ assetId: 'model-1',
+ annotationId: null,
+ config3d: { degree: 360 },
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ }),
+ ).rejects.toThrow('Invalid turntable degree: 360')
+ })
+
+ it('should extract frame with overlaid markups when annotationId is provided on 3D asset', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ projectId: 'proj-1',
+ media: {
+ proxyType: '3d',
+ videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
+ },
+ } as unknown as Asset)
+
+ vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
+ annotations: [
+ {
+ type: 'freehand',
+ color: '#ff0000',
+ points: [
+ [0, 0],
+ [10, 10],
+ ],
+ },
+ ],
+ timestamp: 0.5, // 0.5s = 45°
+ })
+
+ vi.mocked(prisma.workflowTask.create).mockResolvedValue({
+ id: 'task-shot-ann',
+ } as unknown as WorkflowTask)
+
+ vi.mocked(workflowService.executeWait).mockResolvedValue({
+ id: 'task-shot-ann',
+ status: WorkflowTaskStatus.completed,
+ output: {
+ screenshots: [{ key: 'proxy/shot-annotated.webp', timestamp: 0.5 }],
+ },
+ } as unknown as WorkflowTask)
+
+ vi.mocked(s3Service.getObject).mockResolvedValue({
+ buffer: Buffer.from('fake-shot-ann-bytes'),
+ contentType: 'image/webp',
+ } as unknown as { buffer: Buffer; contentType: string })
+
+ const tool = createReadAssetTool('user-1')
+ const result = await tool.execute('call-3d-ann', {
+ assetId: 'model-1',
+ annotationId: 'comm-1',
+ config3d: null,
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ })
+
+ expect(prisma.workflowTask.create).toHaveBeenCalledWith({
+ data: expect.objectContaining({
+ type: WorkflowTaskType.transcode_screenshot,
+ assetId: 'model-1',
+ payload: {
+ projectId: 'proj-1',
+ screenshot: {
+ start: 0.5,
+ end: 0.5,
+ count: 1,
+ commentTimestamp: 0.5,
+ annotations: [
+ {
+ type: 'freehand',
+ color: '#ff0000',
+ points: [
+ [0, 0],
+ [10, 10],
+ ],
+ },
+ ],
+ },
+ },
+ }),
+ })
+
+ const textItem = result.content[0] as { type: 'text'; text: string }
+ expect(textItem.text).toContain('view at 45° with visual markups overlaid:')
+ expect(result.details.key).toBe('proxy/shot-annotated.webp')
+ })
+
+ it('should extract turntable frame at resolved degree angle for 3D session annotations', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ projectId: 'proj-1',
+ media: {
+ proxyType: '3d',
+ videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
+ },
+ } as unknown as Asset)
+
+ const expectedTime = degreeToSecond(90)
+ vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
+ annotations: [
+ {
+ type: 'box',
+ color: '#00ff00',
+ points: [
+ [10, 20],
+ [40, 60],
+ ],
+ },
+ ],
+ timestamp: expectedTime,
+ })
+
+ vi.mocked(prisma.workflowTask.create).mockResolvedValue({
+ id: 'task-shot-3d-session',
+ } as unknown as WorkflowTask)
+
+ vi.mocked(workflowService.executeWait).mockResolvedValue({
+ id: 'task-shot-3d-session',
+ status: WorkflowTaskStatus.completed,
+ output: {
+ screenshots: [{ key: 'proxy/shot-annotated-90.webp', timestamp: expectedTime }],
+ },
+ } as unknown as WorkflowTask)
+
+ vi.mocked(s3Service.getObject).mockResolvedValue({
+ buffer: Buffer.from('fake-shot-90-bytes'),
+ contentType: 'image/webp',
+ } as unknown as { buffer: Buffer; contentType: string })
+
+ const tool = createReadAssetTool('user-1')
+ const result = await tool.execute('call-3d-session-ann', {
+ assetId: 'model-1',
+ annotationId: 'session-entry-1',
+ config3d: null,
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ })
+
+ expect(prisma.workflowTask.create).toHaveBeenCalledWith({
+ data: expect.objectContaining({
+ type: WorkflowTaskType.transcode_screenshot,
+ assetId: 'model-1',
+ payload: {
+ projectId: 'proj-1',
+ screenshot: {
+ start: expectedTime,
+ end: expectedTime,
+ count: 1,
+ commentTimestamp: expectedTime,
+ annotations: [
+ {
+ type: 'box',
+ color: '#00ff00',
+ points: [
+ [10, 20],
+ [40, 60],
+ ],
+ },
+ ],
+ },
+ },
+ }),
+ })
+
+ const textItem = result.content[0] as { type: 'text'; text: string }
+ expect(textItem.text).toContain('view at 90° with visual markups overlaid:')
+ expect(result.details.key).toBe('proxy/shot-annotated-90.webp')
+ })
+
+ it('should throw error if config3d.degree conflicts with annotation angle', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ projectId: 'proj-1',
+ media: {
+ proxyType: '3d',
+ videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
+ },
+ } as unknown as Asset)
+
+ vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
+ annotations: [
+ {
+ type: 'box',
+ color: '#ff0000',
+ points: [
+ [0, 0],
+ [10, 10],
+ ],
+ },
+ ],
+ timestamp: 0.5, // 45°
+ })
+
+ const tool = createReadAssetTool('user-1')
+ await expect(
+ tool.execute('call-3d-conflict', {
+ assetId: 'model-1',
+ annotationId: 'comm-1',
+ config3d: { degree: 90 }, // Conflicts with 45°
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ }),
+ ).rejects.toThrow(
+ 'Annotation "comm-1" was drawn at 45°, but degree 90° was specified. Do not pass a conflicting degree with annotationId.',
+ )
+ })
+
+ it('should throw error if 3D asset turntable video is missing when angle inspection is requested', async () => {
+ vi.mocked(authzService.hasPermission).mockResolvedValue()
+ vi.mocked(prisma.asset.findUnique).mockResolvedValue({
+ id: 'model-1',
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ media: {
+ proxyType: '3d',
+ videoTranscodes: [],
+ },
+ } as unknown as Asset)
+
+ const tool = createReadAssetTool('user-1')
+ await expect(
+ tool.execute('call-3d-missing-turntable', {
+ assetId: 'model-1',
+ annotationId: null,
+ config3d: { degree: 45 },
+ imageConfig: null,
+ videoConfig: null,
+ docConfig: null,
+ }),
+ ).rejects.toThrow('turntable video (turntable.mp4) is missing')
+ })
+ })
+
describe('Video Assets', () => {
it('should throw error if videoConfig is missing for video assets', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
diff --git a/packages/agent/src/tools/read-asset.ts b/packages/agent/src/tools/read-asset.ts
index 9328a79a..b7f09b6c 100644
--- a/packages/agent/src/tools/read-asset.ts
+++ b/packages/agent/src/tools/read-asset.ts
@@ -7,6 +7,12 @@ import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { resolveAnnotationsById } from './annotation-resolver'
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
+import { is3dModel } from '@shumai/core/src/utils/mime'
+import {
+ isValidTurntableDegree,
+ secondToDegree,
+ degreeToSecond,
+} from '@shumai/core/src/utils/model3d'
export const readAssetSchema = Type.Object(
{
@@ -37,6 +43,31 @@ export const readAssetSchema = Type.Object(
),
Type.Null(),
]),
+ config3d: Type.Optional(
+ Type.Union([
+ Type.Object(
+ {
+ degree: Type.Optional(
+ Type.Union([
+ Type.Integer({
+ minimum: 0,
+ maximum: 345,
+ description:
+ 'Turntable rotation degree (integer: 0, 15, 30, ..., 345 in 15° increments) to inspect a specific angle of a 3D model. Leave null/omitted to view the 4-view composite overview.',
+ }),
+ Type.Null(),
+ ]),
+ ),
+ },
+ {
+ additionalProperties: false,
+ description:
+ 'Configuration for 3D model assets. Pass null if inspecting another asset type. Optionally pass degree to inspect a specific turntable angle, or leave null/omitted to view the 4-view composite overview.',
+ },
+ ),
+ Type.Null(),
+ ]),
+ ),
videoConfig: Type.Union([
Type.Object(
{
@@ -167,12 +198,12 @@ export function createReadAssetTool(
label: 'Read Asset',
description:
'Inspects and reads the content of an asset in the workspace. ' +
- 'Supports visual analysis for images, video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
- 'Pass assetId, and provide the specific config (imageConfig, videoConfig, or docConfig) while setting unused configs to null. ' +
+ 'Supports visual analysis for images, 3D models (overview or specific degree angle), video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
+ 'Pass assetId, and provide the specific config (imageConfig, config3d, videoConfig, or docConfig) while setting unused configs to null. ' +
'Optionally provide annotationId to view visual markups drawn on the asset.',
parameters: readAssetSchema,
execute: async (_toolCallId, params) => {
- const { assetId, annotationId, s3KeyOnly, videoConfig, docConfig } = params
+ const { assetId, annotationId, s3KeyOnly, config3d, videoConfig, docConfig } = params
if (agentType === 'autofill' && !userId) {
if (!targetAssetId) {
@@ -243,6 +274,209 @@ export function createReadAssetTool(
const filename = asset.name?.toLowerCase() || ''
const bucket = process.env.S3_BUCKET || 'shumai'
+ // ----------------------------------------------------------------------
+ // Branch 0: 3D Assets (Orthogonal 4-View Composite Image or Degree Angle)
+ // ----------------------------------------------------------------------
+ const is3d = proxyType === '3d' || is3dModel(mediaType, filename)
+ if (is3d) {
+ if (videoConfig) {
+ throw new Error(
+ `Asset ${asset.id} is a 3D model. To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig.`,
+ )
+ }
+
+ if (config3d?.degree !== undefined && config3d?.degree !== null) {
+ if (!isValidTurntableDegree(config3d.degree)) {
+ throw new Error(
+ `Invalid turntable degree: ${config3d.degree}. Degree must be an integer multiple of 15 between 0 and 345 (e.g. 0, 15, 30, 45, ..., 345).`,
+ )
+ }
+ }
+
+ let annotations: PrismaJson.AnnotationList | null = null
+ let commentTimestamp: number | null = null
+ if (annotationId) {
+ const resolved = await resolveAnnotationsById(asset.id, annotationId)
+ annotations = resolved.annotations
+ commentTimestamp = resolved.timestamp
+ if (commentTimestamp !== null && commentTimestamp !== undefined) {
+ const commentDegree = secondToDegree(commentTimestamp)
+ if (
+ config3d?.degree !== undefined &&
+ config3d?.degree !== null &&
+ config3d.degree !== commentDegree
+ ) {
+ throw new Error(
+ `Annotation "${annotationId}" was drawn at ${commentDegree}°, but degree ${config3d.degree}° was specified. Do not pass a conflicting degree with annotationId.`,
+ )
+ }
+ }
+ }
+
+ if ((config3d?.degree !== undefined && config3d?.degree !== null) || annotationId) {
+ if (!mediaInfo?.videoTranscodes || mediaInfo.videoTranscodes.length === 0) {
+ throw new Error(
+ `Asset ${asset.id} is a 3D model, but its turntable video (turntable.mp4) is missing.`,
+ )
+ }
+
+ let targetDegree: number
+ let targetSecond: number
+
+ if (config3d?.degree !== undefined && config3d?.degree !== null) {
+ targetDegree = config3d.degree
+ targetSecond = degreeToSecond(targetDegree)
+ } else {
+ targetSecond =
+ commentTimestamp !== null && commentTimestamp !== undefined ? commentTimestamp : 0
+ targetDegree = secondToDegree(targetSecond)
+ }
+
+ const task = await prisma.workflowTask.create({
+ data: {
+ assetId: asset.id,
+ projectId: asset.projectId || 'none',
+ type: WorkflowTaskType.transcode_screenshot,
+ status: WorkflowTaskStatus.pending,
+ payload: {
+ projectId: asset.projectId || 'none',
+ screenshot: {
+ start: targetSecond,
+ end: targetSecond,
+ count: 1,
+ commentTimestamp: targetSecond,
+ annotations: annotations ?? undefined,
+ },
+ },
+ },
+ })
+
+ const completedTask = await workflowService.executeWait(task)
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode screenshot output
+ const output = completedTask.output as any
+ const screenshots = output?.screenshots as
+ | Array<{ key: string; timestamp: number }>
+ | undefined
+
+ if (!screenshots || screenshots.length === 0) {
+ throw new Error(
+ `Failed to extract 3D turntable frame for asset ${asset.id} at ${targetDegree}°.`,
+ )
+ }
+
+ const shotKey = screenshots[0].key
+ const downloadUrl = await s3Service.presign(
+ bucket,
+ shotKey,
+ 'GET',
+ true,
+ asset.name || undefined,
+ )
+
+ const content: Array = []
+ const hasMarkup = Boolean(annotations && annotations.length > 0)
+ const textDescription = hasMarkup
+ ? `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}° with visual markups overlaid:`
+ : `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}°:`
+
+ if (s3KeyOnly) {
+ content.push({
+ type: 'text',
+ text: textDescription,
+ })
+ } else {
+ const { buffer, contentType } = await s3Service.getObject(bucket, shotKey)
+ const actualMimeType =
+ contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp'
+ content.push({
+ type: 'text',
+ text: textDescription,
+ })
+ content.push({
+ type: 'image',
+ data: buffer.toString('base64'),
+ mimeType: actualMimeType,
+ })
+ }
+
+ return {
+ content,
+ details: {
+ assetId: asset.id,
+ name: asset.name,
+ mediaType: asset.mediaType,
+ key: shotKey,
+ downloadUrl,
+ sourceKeys: [shotKey],
+ },
+ }
+ }
+
+ if (!mediaInfo?.imageTranscodes || mediaInfo.imageTranscodes.length === 0) {
+ throw new Error(
+ `Asset ${asset.id} is a 3D model, but its 4-views image (views.webp) is missing.`,
+ )
+ }
+
+ const mediaKey = mediaInfo.imageTranscodes[0].key
+ if (!mediaKey) {
+ throw new Error(`Asset ${asset.id} is a 3D model, but its 4-views image key is missing.`)
+ }
+
+ const keyToUse = mediaKey
+ const mimeType = 'image/webp'
+
+ const downloadUrl = await s3Service.presign(
+ bucket,
+ keyToUse,
+ 'GET',
+ true,
+ asset.name || undefined,
+ )
+
+ const content: Array = []
+ const textDescription = [
+ `3D asset "${asset.name}" (ID: ${asset.id}, MIME: ${mimeType}, S3 Key: "${keyToUse}"):`,
+ 'Orthogonal 4-view composite image (2x2 grid across 360° rotation):',
+ '- Top-Left quadrant: Front view (0°)',
+ '- Top-Right quadrant: Right side view (90°)',
+ '- Bottom-Left quadrant: Back view (180°)',
+ '- Bottom-Right quadrant: Left side view (270°)',
+ ].join('\n')
+
+ if (s3KeyOnly) {
+ content.push({
+ type: 'text',
+ text: textDescription,
+ })
+ } else {
+ const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
+ const actualMimeType =
+ contentType && contentType !== 'application/octet-stream' ? contentType : mimeType
+ content.push({
+ type: 'text',
+ text: textDescription,
+ })
+ content.push({
+ type: 'image',
+ data: buffer.toString('base64'),
+ mimeType: actualMimeType,
+ })
+ }
+
+ return {
+ content,
+ details: {
+ assetId: asset.id,
+ name: asset.name,
+ mediaType: asset.mediaType,
+ key: keyToUse,
+ downloadUrl,
+ sourceKeys: [keyToUse],
+ },
+ }
+ }
+
// ----------------------------------------------------------------------
// Branch 1: Image Assets
// ----------------------------------------------------------------------
diff --git a/packages/agent/src/tools/read-thread.test.ts b/packages/agent/src/tools/read-thread.test.ts
index a4fbb6f3..51e20346 100644
--- a/packages/agent/src/tools/read-thread.test.ts
+++ b/packages/agent/src/tools/read-thread.test.ts
@@ -129,6 +129,42 @@ describe('createReadThreadTool', () => {
expect(textContent).toContain('- [Ai Agent] (id: reply-1, has_markup: false): Agent answer')
})
+ it('should format 3d asset comment timestamps as degree: X°', async () => {
+ vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
+ id: 'root-3d',
+ message: 'Notice this angle',
+ creator: { name: 'Alice' },
+ second: 0.5,
+ annotation: [{ type: 'box' }],
+ asset: {
+ media: { proxyType: '3d' },
+ mediaType: 'model/gltf-binary',
+ name: 'robot.glb',
+ },
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ } as any)
+ vi.mocked(prisma.assetComment.findMany).mockResolvedValue([
+ {
+ id: 'reply-3d-1',
+ message: 'Looking at back side',
+ creator: { name: 'Bob' },
+ second: 2.0,
+ },
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ ] as any)
+
+ const tool = createReadThreadTool()
+ const result = await tool.execute('call-3d', { threadId: 'root-3d' })
+ const textContent = (result.content[0] as { text: string }).text
+
+ expect(textContent).toContain(
+ 'Thread Root [Alice] (id: root-3d, degree: 45°, has_markup: true): Notice this angle',
+ )
+ expect(textContent).toContain(
+ '- [Bob] (id: reply-3d-1, degree: 180°, has_markup: false): Looking at back side',
+ )
+ })
+
it('should describe threadId parameter as referencing in ', () => {
const tool = createReadThreadTool()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
diff --git a/packages/agent/src/tools/read-thread.ts b/packages/agent/src/tools/read-thread.ts
index 4a258c95..f3c22986 100644
--- a/packages/agent/src/tools/read-thread.ts
+++ b/packages/agent/src/tools/read-thread.ts
@@ -1,6 +1,8 @@
import { prisma } from '@shumai/db'
import { Type } from 'typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
+import { secondToDegree } from '@shumai/core/src/utils/model3d'
+import { getProxyType } from '@shumai/core/src/utils/mime'
const readThreadSchema = Type.Object({
threadId: Type.String({
@@ -9,14 +11,21 @@ const readThreadSchema = Type.Object({
}),
})
-function formatCommentMeta(c: {
- id: string
- second?: number | null
- annotation?: unknown
-}): string {
+function formatCommentMeta(
+ c: {
+ id: string
+ second?: number | null
+ annotation?: unknown
+ },
+ proxyType?: string,
+): string {
const parts = [`id: ${c.id}`]
if (c.second !== null && c.second !== undefined) {
- parts.push(`time: ${c.second}s`)
+ if (proxyType === '3d') {
+ parts.push(`degree: ${secondToDegree(c.second)}°`)
+ } else {
+ parts.push(`time: ${c.second}s`)
+ }
}
const hasMarkup = Boolean(c.annotation && Array.isArray(c.annotation) && c.annotation.length > 0)
parts.push(`has_markup: ${hasMarkup}`)
@@ -34,7 +43,10 @@ export const createReadThreadTool = (): AgentTool<
execute: async (_toolCallId, params) => {
const rootComment = await prisma.assetComment.findUnique({
where: { id: params.threadId },
- include: { creator: true },
+ include: {
+ creator: true,
+ asset: { select: { media: true, mediaType: true, name: true } },
+ },
})
if (!rootComment) {
@@ -61,9 +73,14 @@ export const createReadThreadTool = (): AgentTool<
include: { creator: true },
})
+ const proxyType =
+ (rootComment.asset?.media as PrismaJson.MediaInfo | null)?.proxyType ||
+ getProxyType(rootComment.asset?.mediaType, rootComment.asset?.name) ||
+ undefined
+
const isRootAgent = rootComment.creator?.type === 'agent' || rootComment.sessionId !== null
const rootAuthor = rootComment.creator?.name || (isRootAgent ? 'Ai Agent' : 'User')
- const rootMeta = formatCommentMeta(rootComment)
+ const rootMeta = formatCommentMeta(rootComment, proxyType)
let output = `Thread Root [${rootAuthor}] ${rootMeta}: ${rootComment.message || ''}\n\nReplies (${replies.length}):`
if (replies.length === 0) {
@@ -72,7 +89,7 @@ export const createReadThreadTool = (): AgentTool<
for (const reply of replies) {
const isAgent = reply.creator?.type === 'agent' || reply.sessionId !== null
const author = reply.creator?.name || (isAgent ? 'Ai Agent' : 'User')
- const replyMeta = formatCommentMeta(reply)
+ const replyMeta = formatCommentMeta(reply, proxyType)
output += `\n- [${author}] ${replyMeta}: ${reply.message || ''}`
}
}
diff --git a/packages/agent/src/workflows/agent-chat.test.ts b/packages/agent/src/workflows/agent-chat.test.ts
index 39a90531..7d41b7ef 100644
--- a/packages/agent/src/workflows/agent-chat.test.ts
+++ b/packages/agent/src/workflows/agent-chat.test.ts
@@ -806,4 +806,89 @@ describe('Agent Chat Workflow', () => {
}),
)
})
+
+ it('should format mediaPosition as degree for 3D assets in 1-on-1 chat', async () => {
+ mockActivities.getAssetActivity.mockResolvedValue({
+ id: 'a-3d',
+ project: { teamId: 't1' },
+ type: 'file',
+ name: 'model.glb',
+ mediaType: 'model/gltf-binary',
+ media: { proxyType: '3d' },
+ parentId: null,
+ })
+
+ const task = await prisma.workflowTask.create({
+ data: {
+ type: 'chat',
+ status: 'pending',
+ assetId: 'a-3d',
+ payload: {
+ projectId: 'p1',
+ agent: {
+ agentId: 'b1',
+ prompt: 'Look at this angle',
+ sessionId: 'session-3d-1',
+ second: 0.5, // 0.5s turntable time = 45°
+ },
+ },
+ },
+ })
+
+ await agentChat(task)
+
+ expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
+ expect.objectContaining({
+ messageContext: expect.objectContaining({
+ position: { type: 'degree', degree: 45 },
+ }),
+ }),
+ )
+ })
+
+ it('should format mediaPosition as degree for 3D assets in comment mode', async () => {
+ mockActivities.getAssetActivity.mockResolvedValue({
+ id: 'a-3d',
+ project: { teamId: 't1' },
+ type: 'file',
+ name: 'model.glb',
+ mediaType: 'model/gltf-binary',
+ media: { proxyType: '3d' },
+ parentId: null,
+ })
+
+ mockActivities.getCommentActivity.mockResolvedValue({
+ id: 'comment-3d-1',
+ creatorId: 'user-c1',
+ message: 'Look at the back',
+ replyToId: null,
+ second: 2.0, // 2.0s = 180°
+ attachments: [],
+ })
+
+ const task = await prisma.workflowTask.create({
+ data: {
+ type: 'chat',
+ status: 'pending',
+ assetId: 'a-3d',
+ payload: {
+ projectId: 'p1',
+ agent: {
+ agentId: 'b1',
+ userCommentId: 'comment-3d-1',
+ },
+ },
+ },
+ })
+
+ await agentChat(task)
+
+ expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
+ expect.objectContaining({
+ messageContext: expect.objectContaining({
+ position: { type: 'degree', degree: 180 },
+ }),
+ }),
+ )
+ })
})
diff --git a/packages/agent/src/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts
index 40ea7756..5e205273 100644
--- a/packages/agent/src/workflows/agent-chat.ts
+++ b/packages/agent/src/workflows/agent-chat.ts
@@ -8,6 +8,7 @@ import type {
ShumaiMediaPosition,
} from '@shumai/dtos'
import { getProxyType } from '@shumai/core/src/utils/mime'
+import { secondToDegree } from '@shumai/core/src/utils/model3d'
export async function agentChat(task: WorkflowTask): Promise {
const {
@@ -273,7 +274,9 @@ export async function agentChat(task: WorkflowTask): Promise {
// Build position
let mediaPosition: ShumaiMediaPosition | undefined
if (commentTimestamp !== undefined && commentTimestamp !== null) {
- if (proxyType === 'pdf') {
+ if (proxyType === '3d') {
+ mediaPosition = { type: 'degree', degree: secondToDegree(commentTimestamp) }
+ } else if (proxyType === 'pdf') {
mediaPosition = { type: 'page', page: Math.round(commentTimestamp) }
} else {
mediaPosition = { type: 'time', seconds: commentTimestamp }
diff --git a/packages/agent/src/workflows/agent-embedding.test.ts b/packages/agent/src/workflows/agent-embedding.test.ts
index 2d0b2a84..54bae9d4 100644
--- a/packages/agent/src/workflows/agent-embedding.test.ts
+++ b/packages/agent/src/workflows/agent-embedding.test.ts
@@ -211,6 +211,68 @@ describe('Agent Embedding Workflow', () => {
})
})
+ it('should run agent embedding workflow successfully for 3D asset using 4-views image', async () => {
+ mockActivities.getEmbeddingContextActivity.mockResolvedValue({
+ agent: { id: 'b1' },
+ asset: {
+ id: 'model-1',
+ mediaType: 'model/gltf-binary',
+ storageKey: { key: 'raw/model.glb' },
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [{ key: 'files/views.webp', format: 'webp' }],
+ },
+ },
+ chunkDuration: 60.0,
+ chunkOverlap: 5.0,
+ })
+
+ const task = await prisma.workflowTask.create({
+ data: {
+ type: 'ai_embedding',
+ status: 'pending',
+ assetId: 'model-1',
+ teamId: 't1',
+ },
+ })
+
+ await agentEmbeddingMedia(task)
+
+ expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({
+ teamId: 't1',
+ assetKey: 'files/views.webp',
+ mediaType: 'image/webp',
+ })
+ })
+
+ it('should reject 3D asset for embeddings if imageTranscodes is missing', async () => {
+ mockActivities.getEmbeddingContextActivity.mockResolvedValue({
+ agent: { id: 'b1' },
+ asset: {
+ id: 'model-1',
+ mediaType: 'model/gltf-binary',
+ storageKey: { key: 'raw/model.glb' },
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [],
+ },
+ },
+ })
+
+ const task = await prisma.workflowTask.create({
+ data: {
+ type: 'ai_embedding',
+ status: 'pending',
+ assetId: 'model-1',
+ teamId: 't1',
+ },
+ })
+
+ await expect(agentEmbeddingMedia(task)).rejects.toThrow(
+ 'unsupported media type for embeddings: model/gltf-binary',
+ )
+ })
+
it('should run agent embedding workflow successfully for video in chunks', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts
index ab365ab7..23cb747d 100644
--- a/packages/agent/src/workflows/agent-embedding.ts
+++ b/packages/agent/src/workflows/agent-embedding.ts
@@ -70,7 +70,10 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const { asset, chunkDuration, chunkOverlap } = context as any
- const isImage = asset.mediaType?.startsWith('image/')
+ const mediaInfo = asset.media as PrismaJson.MediaInfo | null
+ const is3dWithViews =
+ mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0
+ const isImage = asset.mediaType?.startsWith('image/') || is3dWithViews
const isVideo = asset.mediaType?.startsWith('video/')
if (!isImage && !isVideo) {
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 7c162257..76373a87 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -32,4 +32,5 @@ export * from './media-generation/media-generation'
export * from './sandbox'
export * from './asset/comments-export'
export * from './turntable/turntable'
+export * from './utils/model3d'
export * from './integrations/helper'
diff --git a/packages/core/src/transcode/transcode.test.ts b/packages/core/src/transcode/transcode.test.ts
index 802c5086..ed9fb94d 100644
--- a/packages/core/src/transcode/transcode.test.ts
+++ b/packages/core/src/transcode/transcode.test.ts
@@ -4432,6 +4432,89 @@ describe('TranscodeService', () => {
})
})
+ describe('generate3dViews', () => {
+ it('executes ffmpeg with 2x2 grid filter, libwebp encoder, and q=85', async () => {
+ let executedArgs: string[] = []
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ ;(child_process.execFile as any).mockImplementation(
+ (
+ _cmd: string,
+ args: string[],
+ cb: (err: unknown, result: { stdout: string; stderr: string }) => void,
+ ) => {
+ executedArgs = args
+ cb(null, { stdout: '', stderr: '' })
+ },
+ )
+
+ const inputPath = path.join(tempDir, 'turntable.mp4')
+ const outputPath = path.join(tempDir, 'views.webp')
+
+ await transcodeService.generate3dViews(inputPath, outputPath)
+
+ expect(child_process.execFile).toHaveBeenCalledWith(
+ 'ffmpeg',
+ expect.any(Array),
+ expect.any(Function),
+ )
+ expect(executedArgs).toEqual([
+ '-y',
+ '-loglevel',
+ 'warning',
+ '-i',
+ inputPath,
+ '-filter_complex',
+ "[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]",
+ '-map',
+ '[views_out]',
+ '-frames:v',
+ '1',
+ '-c:v',
+ 'libwebp',
+ '-q:v',
+ '85',
+ outputPath,
+ ])
+ })
+
+ it('passes abort signal to execFile when provided', async () => {
+ const controller = new AbortController()
+ let optionsPassed: unknown = null
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ ;(child_process.execFile as any).mockImplementation(
+ (_cmd: string, _args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
+ const cb = (typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb) as (
+ err: unknown,
+ result: { stdout: string; stderr: string },
+ ) => void
+ if (typeof optionsOrCb !== 'function') {
+ optionsPassed = optionsOrCb
+ }
+ cb(null, { stdout: '', stderr: '' })
+ },
+ )
+
+ const inputPath = path.join(tempDir, 'turntable.mp4')
+ const outputPath = path.join(tempDir, 'views.webp')
+
+ await transcodeService.generate3dViews(inputPath, outputPath, controller.signal)
+
+ expect(optionsPassed).toMatchObject({ signal: controller.signal })
+ })
+
+ it('rejects with cancellation error if aborted before execution', async () => {
+ const controller = new AbortController()
+ controller.abort()
+
+ const inputPath = path.join(tempDir, 'aborted.mp4')
+ const outputPath = path.join(tempDir, 'views.webp')
+
+ await expect(
+ transcodeService.generate3dViews(inputPath, outputPath, controller.signal),
+ ).rejects.toThrow('3D views generation cancelled')
+ })
+ })
+
describe('HLS utilities', () => {
it('builds master playlist correctly with bandwidth and codecs', () => {
const playlist = buildHlsMasterPlaylist([
diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts
index a80eb924..ccf197d7 100644
--- a/packages/core/src/transcode/transcode.ts
+++ b/packages/core/src/transcode/transcode.ts
@@ -3192,6 +3192,37 @@ export class TranscodeService {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
}
+
+ async generate3dViews(
+ inputFile: string,
+ outputViews: string,
+ signal?: AbortSignal,
+ ): Promise {
+ if (signal?.aborted) {
+ throw new Error('3D views generation cancelled')
+ }
+ const filterComplex = `[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]`
+ const args = [
+ '-i',
+ inputFile,
+ '-filter_complex',
+ filterComplex,
+ '-map',
+ '[views_out]',
+ '-frames:v',
+ '1',
+ '-c:v',
+ 'libwebp',
+ '-q:v',
+ '85',
+ outputViews,
+ ]
+ if (signal) {
+ await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
+ } else {
+ await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
+ }
+ }
}
function renderAnnotationsToSvg(
diff --git a/packages/core/src/utils/model3d.test.ts b/packages/core/src/utils/model3d.test.ts
new file mode 100644
index 00000000..ff49cf6e
--- /dev/null
+++ b/packages/core/src/utils/model3d.test.ts
@@ -0,0 +1,90 @@
+import { describe, expect, it } from 'vitest'
+import {
+ calculateTurntableFrameCenterTime,
+ degreeToSecond,
+ degreeToTurntableFrame,
+ isValidTurntableDegree,
+ secondToDegree,
+ secondToTurntableFrame,
+ TOTAL_TURNTABLE_FRAMES,
+ TURNTABLE_FPS,
+} from './model3d'
+
+describe('model3d utils', () => {
+ describe('secondToTurntableFrame', () => {
+ it('converts playback seconds to frame indices (0..23)', () => {
+ expect(secondToTurntableFrame(0)).toBe(0)
+ expect(secondToTurntableFrame(0.167)).toBe(1)
+ expect(secondToTurntableFrame(0.5)).toBe(3)
+ expect(secondToTurntableFrame(2.0)).toBe(12)
+ expect(secondToTurntableFrame(3.9)).toBe(23)
+ })
+
+ it('clamps out-of-range seconds', () => {
+ expect(secondToTurntableFrame(-1)).toBe(0)
+ expect(secondToTurntableFrame(10)).toBe(TOTAL_TURNTABLE_FRAMES - 1)
+ })
+ })
+
+ describe('secondToDegree', () => {
+ it('converts seconds to turntable degrees (0, 15, ..., 345)', () => {
+ expect(secondToDegree(0)).toBe(0)
+ expect(secondToDegree(1 / TURNTABLE_FPS)).toBe(15)
+ expect(secondToDegree(0.5)).toBe(45)
+ expect(secondToDegree(2.0)).toBe(180)
+ expect(secondToDegree(3.9)).toBe(345)
+ })
+ })
+
+ describe('degreeToTurntableFrame', () => {
+ it('converts discrete degrees to frame indices', () => {
+ expect(degreeToTurntableFrame(0)).toBe(0)
+ expect(degreeToTurntableFrame(15)).toBe(1)
+ expect(degreeToTurntableFrame(45)).toBe(3)
+ expect(degreeToTurntableFrame(90)).toBe(6)
+ expect(degreeToTurntableFrame(180)).toBe(12)
+ expect(degreeToTurntableFrame(345)).toBe(23)
+ })
+
+ it('normalizes degrees outside [0, 360)', () => {
+ expect(degreeToTurntableFrame(360)).toBe(0)
+ expect(degreeToTurntableFrame(-15)).toBe(23)
+ expect(degreeToTurntableFrame(720)).toBe(0)
+ })
+ })
+
+ describe('calculateTurntableFrameCenterTime and degreeToSecond', () => {
+ it('calculates frame center times within bounds', () => {
+ const centerTime0 = calculateTurntableFrameCenterTime(0)
+ expect(centerTime0).toBeCloseTo(0.5 / TURNTABLE_FPS)
+
+ const centerTime23 = calculateTurntableFrameCenterTime(23)
+ expect(centerTime23).toBeLessThan(TOTAL_TURNTABLE_FRAMES / TURNTABLE_FPS)
+ expect(centerTime23).toBeGreaterThan(23 / TURNTABLE_FPS)
+
+ expect(degreeToSecond(0)).toBeCloseTo(centerTime0)
+ expect(degreeToSecond(45)).toBeCloseTo((3 + 0.5) / TURNTABLE_FPS)
+ expect(degreeToSecond(345)).toBeCloseTo(centerTime23)
+ })
+ })
+
+ describe('isValidTurntableDegree', () => {
+ it('returns true for integer multiples of 15 from 0 to 345', () => {
+ for (let deg = 0; deg < 360; deg += 15) {
+ expect(isValidTurntableDegree(deg)).toBe(true)
+ }
+ })
+
+ it('returns false for non-multiples of 15', () => {
+ expect(isValidTurntableDegree(10)).toBe(false)
+ expect(isValidTurntableDegree(44)).toBe(false)
+ expect(isValidTurntableDegree(45.5)).toBe(false)
+ })
+
+ it('returns false for out of range or negative degrees', () => {
+ expect(isValidTurntableDegree(-15)).toBe(false)
+ expect(isValidTurntableDegree(360)).toBe(false)
+ expect(isValidTurntableDegree(720)).toBe(false)
+ })
+ })
+})
diff --git a/packages/core/src/utils/model3d.ts b/packages/core/src/utils/model3d.ts
new file mode 100644
index 00000000..0c32cf4b
--- /dev/null
+++ b/packages/core/src/utils/model3d.ts
@@ -0,0 +1,58 @@
+export const TOTAL_TURNTABLE_FRAMES = 24
+export const TURNTABLE_FPS = 6
+export const TURNTABLE_FRAME_DURATION = 1 / TURNTABLE_FPS
+
+/**
+ * Converts a playback second into a turntable frame index (0..23), clamped.
+ */
+export function secondToTurntableFrame(second: number): number {
+ return Math.min(
+ Math.max(0, Math.floor(second * TURNTABLE_FPS + 1e-4)),
+ TOTAL_TURNTABLE_FRAMES - 1,
+ )
+}
+
+/**
+ * Converts a turntable playback second to turntable degree (0, 15, 30, ..., 345).
+ */
+export function secondToDegree(second: number): number {
+ const frameIndex = secondToTurntableFrame(second)
+ return (frameIndex * 15) % 360
+}
+
+/**
+ * Calculates the presentation time within a turntable frame.
+ * Seeking near frame center avoids PTS boundary inaccuracies in video decoders.
+ */
+export function calculateTurntableFrameCenterTime(frameIndex: number): number {
+ const normalizedFrame =
+ ((frameIndex % TOTAL_TURNTABLE_FRAMES) + TOTAL_TURNTABLE_FRAMES) % TOTAL_TURNTABLE_FRAMES
+ if (normalizedFrame === TOTAL_TURNTABLE_FRAMES - 1) {
+ return normalizedFrame * TURNTABLE_FRAME_DURATION + TURNTABLE_FRAME_DURATION * 0.25
+ }
+ return (normalizedFrame + 0.5) * TURNTABLE_FRAME_DURATION
+}
+
+/**
+ * Converts a rotation degree (0..345) to a turntable frame index (0..23).
+ */
+export function degreeToTurntableFrame(degree: number): number {
+ const normalizedDegree = ((Math.round(degree) % 360) + 360) % 360
+ return Math.round(normalizedDegree / 15) % TOTAL_TURNTABLE_FRAMES
+}
+
+/**
+ * Converts a rotation degree into a turntable video playback second (center time of that frame).
+ */
+export function degreeToSecond(degree: number): number {
+ const frameIndex = degreeToTurntableFrame(degree)
+ return calculateTurntableFrameCenterTime(frameIndex)
+}
+
+/**
+ * Validates whether a degree is a valid discrete turntable angle:
+ * integer multiple of 15 between 0 and 345 inclusive (0, 15, 30, ..., 345).
+ */
+export function isValidTurntableDegree(degree: number): boolean {
+ return Number.isInteger(degree) && degree >= 0 && degree <= 345 && degree % 15 === 0
+}
diff --git a/packages/dtos/src/chat.ts b/packages/dtos/src/chat.ts
index 1159ba5a..2705dd16 100644
--- a/packages/dtos/src/chat.ts
+++ b/packages/dtos/src/chat.ts
@@ -47,7 +47,7 @@ export const shumaiAssetContextSchema = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
- mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(),
+ mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(),
mimeType: z.string().optional(),
parentId: z.string().optional(),
path: z.string().optional(),
@@ -64,7 +64,7 @@ export const shumaiAttachedFileContextSchema = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
- mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(),
+ mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(),
mimeType: z.string().optional(),
path: z.string().optional(),
url: z.string().optional(),
@@ -75,6 +75,7 @@ export type ShumaiAttachedFileContext = z.infer
diff --git a/packages/dtos/src/context.ts b/packages/dtos/src/context.ts
index 2570f0e9..e8192e10 100644
--- a/packages/dtos/src/context.ts
+++ b/packages/dtos/src/context.ts
@@ -104,6 +104,8 @@ export function serializeContextToXml(context?: ShumaiMessageContext | null): st
lines.push(` `)
} else if (context.position.type === 'page') {
lines.push(` `)
+ } else if (context.position.type === 'degree') {
+ lines.push(` `)
}
}
diff --git a/packages/transcode/src/activities/transcode-3d.test.ts b/packages/transcode/src/activities/transcode-3d.test.ts
index 72c2a34f..ad22225b 100644
--- a/packages/transcode/src/activities/transcode-3d.test.ts
+++ b/packages/transcode/src/activities/transcode-3d.test.ts
@@ -4,6 +4,7 @@ import {
start3dRenderActivity,
waitFor3dVideoRenderActivity,
generate3dSpriteActivity,
+ generate3dViewsActivity,
} from './transcode'
import { turntableService } from '@shumai/core/src/turntable/turntable'
import { s3Service } from '@shumai/core/src/s3/s3'
@@ -31,6 +32,7 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({
transcodeService: {
convertImageToWebp: vi.fn(),
generate3dSprite: vi.fn(),
+ generate3dViews: vi.fn(),
},
}))
@@ -55,6 +57,7 @@ describe('3D Transcode Activities', () => {
vi.mocked(fs.readFileSync).mockReturnValue(Buffer.from('mock-bytes'))
vi.mocked(transcodeService.convertImageToWebp).mockResolvedValue(Buffer.from('webp-bytes'))
vi.mocked(transcodeService.generate3dSprite).mockResolvedValue()
+ vi.mocked(transcodeService.generate3dViews).mockResolvedValue()
vi.mocked(s3Service.putObject).mockResolvedValue({} as never)
vi.mocked(turntableService.deleteTask).mockResolvedValue(true)
})
@@ -324,4 +327,29 @@ describe('3D Transcode Activities', () => {
expect(res.spriteKey).toBe('files/asset-1/sprite.webp')
})
})
+
+ describe('generate3dViewsActivity', () => {
+ it('generates 3d 4-views composite image and uploads to S3', async () => {
+ const res = await generate3dViewsActivity({
+ taskId: 'wf-task-4',
+ assetKey: 'files/asset-1/model.glb',
+ videoFilePath: '/tmp/turntable.mp4',
+ viewsKey: 'files/asset-1/views.webp',
+ })
+
+ expect(transcodeService.generate3dViews).toHaveBeenCalledWith(
+ '/tmp/turntable.mp4',
+ '/tmp/views.webp',
+ undefined,
+ )
+ expect(s3Service.putObject).toHaveBeenCalledWith(
+ 'shumai',
+ 'files/asset-1/views.webp',
+ expect.any(Buffer),
+ expect.any(Number),
+ 'image/webp',
+ )
+ expect(res.viewsKey).toBe('files/asset-1/views.webp')
+ })
+ })
})
diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts
index 9a39c250..afd8fcf2 100644
--- a/packages/transcode/src/activities/transcode.test.ts
+++ b/packages/transcode/src/activities/transcode.test.ts
@@ -27,6 +27,7 @@ import {
transcodeVideoChunkActivity,
deleteS3ObjectActivity,
createAutofillTaskIfEnabledActivity,
+ createEmbeddingTaskIfEnabledActivity,
} from './transcode'
vi.mock('@shumai/core/src/s3/s3', () => ({
@@ -1717,4 +1718,115 @@ describe('Transcode Activities', () => {
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
})
})
+
+ describe('createEmbeddingTaskIfEnabledActivity', () => {
+ it('should create ai_embedding task for 3D asset with imageTranscodes', async () => {
+ const user = await prisma.user.create({
+ data: { name: 'Test User Embedding', email: 'test-embedding-agent@test.com' },
+ })
+ const team = await prisma.team.create({
+ data: { name: 'Test Team Embedding' },
+ })
+ await prisma.teamMember.create({
+ data: { teamId: team.id, userId: user.id, role: 'owner' },
+ })
+ const project = await prisma.project.create({
+ data: { name: 'Test Project Embedding', teamId: team.id },
+ })
+ const embeddingAgent = await prisma.agent.create({
+ data: {
+ id: user.id,
+ type: 'embedding',
+ enabled: true,
+ teamId: team.id,
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ config: { provider: 'test', model: 'test' } as any,
+ },
+ })
+ const asset = await prisma.asset.create({
+ data: {
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ type: 'file',
+ status: 'processed',
+ projectId: project.id,
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [
+ {
+ key: 'files/asset-1/views.webp',
+ width: 2880,
+ height: 2880,
+ quality: 85,
+ format: 'webp',
+ },
+ ],
+ } as unknown as PrismaJson.MediaInfo,
+ },
+ })
+
+ await createEmbeddingTaskIfEnabledActivity({
+ assetId: asset.id,
+ teamId: team.id,
+ projectId: project.id,
+ })
+
+ const task = await prisma.workflowTask.findFirst({
+ where: { assetId: asset.id, type: 'ai_embedding' },
+ })
+ expect(task).toBeDefined()
+ expect(task?.status).toBe('pending')
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ expect((task?.payload as any)?.agent?.agentId).toBe(embeddingAgent.id)
+ })
+
+ it('should NOT create ai_embedding task for 3D asset without imageTranscodes', async () => {
+ const user = await prisma.user.create({
+ data: { name: 'Test User Embedding 2', email: 'test-embedding-agent-2@test.com' },
+ })
+ const team = await prisma.team.create({
+ data: { name: 'Test Team Embedding 2' },
+ })
+ await prisma.teamMember.create({
+ data: { teamId: team.id, userId: user.id, role: 'owner' },
+ })
+ const project = await prisma.project.create({
+ data: { name: 'Test Project Embedding 2', teamId: team.id },
+ })
+ await prisma.agent.create({
+ data: {
+ id: user.id,
+ type: 'embedding',
+ enabled: true,
+ teamId: team.id,
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ config: { provider: 'test', model: 'test' } as any,
+ },
+ })
+ const asset = await prisma.asset.create({
+ data: {
+ name: 'robot.glb',
+ mediaType: 'model/gltf-binary',
+ type: 'file',
+ status: 'processed',
+ projectId: project.id,
+ media: {
+ proxyType: '3d',
+ imageTranscodes: [],
+ } as unknown as PrismaJson.MediaInfo,
+ },
+ })
+
+ await createEmbeddingTaskIfEnabledActivity({
+ assetId: asset.id,
+ teamId: team.id,
+ projectId: project.id,
+ })
+
+ const task = await prisma.workflowTask.findFirst({
+ where: { assetId: asset.id, type: 'ai_embedding' },
+ })
+ expect(task).toBeNull()
+ })
+ })
})
diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts
index ad67f90f..1f35d5fe 100644
--- a/packages/transcode/src/activities/transcode.ts
+++ b/packages/transcode/src/activities/transcode.ts
@@ -1360,8 +1360,11 @@ export async function createEmbeddingTaskIfEnabledActivity(
return
}
+ const mediaInfo = asset.media as PrismaJson.MediaInfo | null
const isVideo = asset.mediaType?.startsWith('video/')
- const isImage = asset.mediaType?.startsWith('image/')
+ const isImage =
+ asset.mediaType?.startsWith('image/') ||
+ (mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0)
if (!isVideo && !isImage) {
return
}
@@ -1886,3 +1889,51 @@ export async function generate3dSpriteActivity(
})
}
}
+
+export interface Generate3dViewsActivityParams {
+ taskId: string
+ assetKey: string
+ videoFilePath: string
+ viewsKey: string
+}
+
+export async function generate3dViewsActivity(
+ params: Generate3dViewsActivityParams,
+): Promise<{ viewsKey: string }> {
+ const signal = getActivityCancellationSignal(params.taskId)
+ if (signal?.aborted) {
+ throw ApplicationFailure.create({
+ message: '3D views generation cancelled',
+ nonRetryable: true,
+ })
+ }
+
+ const bucket = process.env.S3_BUCKET || 'shumai'
+ const tmpDir = path.dirname(params.videoFilePath)
+ const viewsFilePath = path.join(tmpDir, 'views.webp')
+
+ try {
+ await transcodeService.generate3dViews(params.videoFilePath, viewsFilePath, signal)
+ const viewsBuffer = fs.readFileSync(viewsFilePath)
+
+ await ensureAssetNotPurging(params.assetKey)
+ await s3Service.putObject(
+ bucket,
+ params.viewsKey,
+ viewsBuffer,
+ viewsBuffer.length,
+ 'image/webp',
+ )
+
+ return { viewsKey: params.viewsKey }
+ } catch (err) {
+ if (err instanceof ApplicationFailure) {
+ throw err
+ }
+ const errMsg = err instanceof Error ? err.message : String(err)
+ throw ApplicationFailure.create({
+ message: `Failed to generate 3D views: ${errMsg}`,
+ nonRetryable: true,
+ })
+ }
+}
diff --git a/packages/transcode/src/workflows/transcode-3d.test.ts b/packages/transcode/src/workflows/transcode-3d.test.ts
index 494a08c3..cacbeadd 100644
--- a/packages/transcode/src/workflows/transcode-3d.test.ts
+++ b/packages/transcode/src/workflows/transcode-3d.test.ts
@@ -29,6 +29,7 @@ describe('transcode3dWorkflow', () => {
_activityName: 'waitFor3dVideoRenderActivity',
}),
generate3dSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generate3dSpriteActivity' }),
+ generate3dViewsActivity: Object.assign(vi.fn(), { _activityName: 'generate3dViewsActivity' }),
updateAssetMediaActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetMediaActivity',
}),
@@ -173,6 +174,13 @@ describe('transcode3dWorkflow', () => {
spriteKey: 'files/asset-3d/sprite.webp',
})
+ expect(mockActivities.generate3dViewsActivity).toHaveBeenCalledWith({
+ taskId: 'task-3d',
+ assetKey: 'files/asset-3d/robot.glb',
+ videoFilePath: '/tmp/turntable.mp4',
+ viewsKey: 'files/asset-3d/views.webp',
+ })
+
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith(
expect.objectContaining({
assetId: 'asset-3d',
@@ -182,6 +190,15 @@ describe('transcode3dWorkflow', () => {
frames: 24,
poster: { key: 'files/asset-3d/poster.webp' },
sprite: expect.objectContaining({ key: 'files/asset-3d/sprite.webp' }),
+ imageTranscodes: [
+ {
+ key: 'files/asset-3d/views.webp',
+ width: 2880,
+ height: 2880,
+ quality: 85,
+ format: 'webp',
+ },
+ ],
}),
}),
)
diff --git a/packages/transcode/src/workflows/transcode-3d.ts b/packages/transcode/src/workflows/transcode-3d.ts
index 24597f9f..496b8d02 100644
--- a/packages/transcode/src/workflows/transcode-3d.ts
+++ b/packages/transcode/src/workflows/transcode-3d.ts
@@ -23,6 +23,7 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise {
start3dRenderActivity,
waitFor3dVideoRenderActivity,
generate3dSpriteActivity,
+ generate3dViewsActivity,
updateAssetMediaActivity,
createEmbeddingTaskIfEnabledActivity,
createAutofillTaskIfEnabledActivity,
@@ -46,6 +47,7 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise {
const videoKey = assetDir ? `${assetDir}/turntable.mp4` : 'turntable.mp4'
const posterKey = assetDir ? `${assetDir}/poster.webp` : 'poster.webp'
const spriteKey = assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp'
+ const viewsKey = assetDir ? `${assetDir}/views.webp` : 'views.webp'
const renderStart = await executeActivity(workerQueue, start3dRenderActivity, {
taskId: task.id,
@@ -117,6 +119,23 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise {
tileY: 10,
}
+ await executeActivity(workerQueue, generate3dViewsActivity, {
+ taskId: task.id,
+ assetKey: key,
+ videoFilePath: renderVideo.videoFilePath,
+ viewsKey,
+ })
+
+ mediaInfo.imageTranscodes = [
+ {
+ key: viewsKey,
+ width: 2880,
+ height: 2880,
+ quality: 85,
+ format: 'webp',
+ },
+ ]
+
await executeActivity(workerQueue, updateAssetMediaActivity, {
assetId: asset.id,
mediaInfo,