feat(agent): support 3D degree comments, markup annotations, and config3d in read_asset (#473)

* feat(agent): support AI embedding, autofill, and read_asset for 3D assets

* feat(agent): support 3D degree comments, markup annotations, and config3d in read_asset

* fix(agent): resolve 3D session annotations to turntable degree frame time
This commit is contained in:
Yiling-J
2026-10-13 00:03:21 +08:00
committed by GitHub
parent 0f9d213b1d
commit fe8d404a44
26 changed files with 1687 additions and 21 deletions
+1
View File
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Fixed
* **agent**: Fix an issue where 3D visual markup annotations in 1-on-1 chat sessions were inspected at 0° instead of the user's annotated degree angle
* **transcode**: Fix an issue where portrait photos from cameras and phones appeared sideways in thumbnails, previews, watermarked share links and annotated snapshots, and were listed with their width and height swapped
### Changed
+132
View File
@@ -662,6 +662,23 @@ describe('Agent Activities', () => {
const pdfPrompt = mockHarness.prompt.mock.calls[1]?.[0] ?? ''
expect(pdfPrompt).toContain('Document pages: 50')
expect(pdfPrompt).not.toContain('Video duration')
// For a 3D asset, should produce 3D Model recommendation
await autofillAiActivity({
teamId: 't1',
assetId: 'model-3',
assetName: 'robot.glb',
mediaType: 'model/gltf-binary',
images: [],
fields: [{ id: 'f1', config: { name: 'F1', type: 'text' } }],
context,
})
const modelPrompt = mockHarness.prompt.mock.calls[2]?.[0] ?? ''
expect(modelPrompt).toContain('3D Model: This asset has an orthogonal 4-view composite image')
expect(modelPrompt).toContain(
'Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right',
)
})
})
@@ -1170,6 +1187,43 @@ describe('Agent Database Activities Integration', () => {
expect((rootEntry as any)?.details?.thread).toBeUndefined()
})
it('should set media position to degree for 3D asset comments', async () => {
const asset3d = await prisma.asset.create({
data: {
name: 'robot.glb',
type: 'file',
status: 'processed',
mediaType: 'model/gltf-binary',
media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo,
projectId: project.id,
},
})
const comment3d = await prisma.assetComment.create({
data: {
assetId: asset3d.id,
message: 'Nice angle',
creatorId: user.id,
second: 0.5, // 45°
},
})
const sessionId = await initializeAgentSessionActivity({
teamId: team.id,
agentId: 'test-agent-id',
userCommentId: comment3d.id,
userId: user.id,
})
const storage = new DatabaseSessionStorage(sessionId)
const pathEntries = await storage.getPathToRoot(comment3d.id)
const entry = pathEntries.find((e) => e.id === comment3d.id)
expect(entry).toBeDefined()
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- test entry position parsing
expect((entry as any)?.details?.position).toEqual({ type: 'degree', degree: 45 })
})
it('should return user name and role for team member', async () => {
const info = await getUserTeamInfoActivity({
userId: user.id,
@@ -2105,5 +2159,83 @@ describe('Agent Database Activities Integration', () => {
expect(systemPromptArg).toContain('<thread id="..." reply_count="..." />')
expect(systemPromptArg).toContain('read_thread')
})
it('agentChatActivity injects # 3D Model Inspection into systemPrompt', async () => {
const user = await prisma.user.create({
data: { name: 'User_3d_Prompt', email: `prompt-3d-${Date.now()}@example.com` },
})
const team = await prisma.team.create({
data: { name: 'Team_3d_Prompt_' + Date.now() },
})
const project = await prisma.project.create({
data: { name: 'Project_3d_Prompt', teamId: team.id },
})
const asset = await prisma.asset.create({
data: {
name: 'robot.glb',
type: 'file',
status: 'processed',
mediaType: 'model/gltf-binary',
media: { proxyType: '3d' } as unknown as PrismaJson.MediaInfo,
projectId: project.id,
},
})
const comment = await prisma.assetComment.create({
data: {
assetId: asset.id,
creatorId: user.id,
message: 'Check this 3d model',
},
})
const mockSession = {
id: 'mock-session-id',
subscribe: vi.fn(),
getEntries: vi.fn().mockResolvedValue([]),
getStorage: vi.fn().mockReturnValue({ sessionId: 'mock-session-id' }),
}
const mockHarness = {
subscribe: vi.fn(),
prompt: vi.fn().mockResolvedValue({
content: [{ type: 'text', text: 'Response' }],
usage: { input: 10, output: 20 },
}),
}
vi.mocked(piAgent.createAgentSession).mockResolvedValue({
session: mockSession as unknown as Session<DatabaseSessionMetadata>,
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Mock AgentHarness instance for activity test
harness: mockHarness as unknown as AgentHarness<any, any, any, any>,
})
const context = {
agent: { id: 'agent-1', provider: { name: 'google' }, modelRef: { modelId: 'gemini' } },
dbProviders: [],
teamSkills: [],
allowedDomains: [],
} as unknown as AgentExecutionContext
await agentChatActivity({
teamId: team.id,
agentId: 'agent-1',
message: 'Hello',
imageUrls: [],
projectId: project.id,
folderId: asset.id,
assetId: asset.id,
sessionId: 'mock-session-id',
userCommentId: comment.id,
userId: user.id,
context,
})
const lastCall = vi.mocked(piAgent.createAgentSession).mock.calls.at(-1)
const systemPromptArg = lastCall?.[0].systemPrompt as string
expect(systemPromptArg).toContain('# 3D Model Inspection')
expect(systemPromptArg).toContain('media_type="3d"')
expect(systemPromptArg).toContain('config3d')
expect(systemPromptArg).toContain('<position type="degree" degree="..." />')
})
})
})
+21 -3
View File
@@ -21,6 +21,7 @@ import {
import { aiUsageService } from '@shumai/core/src/ai-usage/ai-usage'
import { quotaService, QuotaExceededError } from '@shumai/core/src/quota/quota-service'
import { getProxyType } from '@shumai/core/src/utils/mime'
import { secondToDegree } from '@shumai/core/src/utils/model3d'
import {
UpdateAssetMetadataRequest,
type ShumaiMessageContext,
@@ -179,6 +180,8 @@ User messages may contain a <context> block detailing the user, active asset loc
systemPrompt += `\n\n# Comment Threads\nWhen operating in comment mode, previous top-level messages in the conversation history may contain a <thread id="..." reply_count="..." /> tag indicating an earlier discussion thread with replies. If a user's question refers to or depends on earlier comments or discussions, you can use the 'read_thread' tool with the thread ID to inspect all replies in that thread.`
}
systemPrompt += `\n\n# 3D Model Inspection\nWhen interacting with 3D model assets (media_type="3d"), you can visually inspect the asset using 'read_asset'. Calling 'read_asset' with config3d: null (or config3d: {}) returns an orthogonal 4-view composite image (Front at top-left, Right side at top-right, Back at bottom-left, Left side at bottom-right). If a <position type="degree" degree="..." /> tag is present or a specific angle is discussed, pass 'config3d: { degree: ... }' (integer 0, 15, 30, ..., 345) to inspect that view angle. If an <annotation id="..." /> tag is present on a 3D asset, pass 'annotationId' to inspect the angle with the user's visual markup drawings overlaid.`
if (agent.type === 'autofill') {
systemPrompt += `\n\n# Autofill Instructions\nWhen performing metadata autofill, inspect the asset content using the 'read_asset' tool. If 'read_asset' fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.`
}
@@ -564,6 +567,7 @@ export async function autofillAiActivity(params: AutofillAiParams) {
const proxyType = getProxyType(params.mediaType, params.assetName)
const isVideoOrAudio = proxyType === 'video' || proxyType === 'audio'
const isPdf = proxyType === 'pdf'
const is3d = proxyType === '3d'
if (params.duration !== undefined && params.duration > 0 && (!proxyType || isVideoOrAudio)) {
promptLines.push(
@@ -575,11 +579,16 @@ export async function autofillAiActivity(params: AutofillAiParams) {
`- Document pages: ${params.pageCount}. Recommended inspection: call read_asset with docConfig: { mode: "pages", startPage: 1, endPage: ${Math.min(params.pageCount, 20)} } or mode: "text".`,
)
}
if (is3d) {
promptLines.push(
'- 3D Model: This asset has an orthogonal 4-view composite image (2x2 grid showing Front view at top-left, Right side view at top-right, Back view at bottom-left, Left side view at bottom-right). Call read_asset to inspect the 4 views.',
)
}
promptLines.push(
'',
'Instructions:',
`1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type).`,
`1. Call the "read_asset" tool with assetId: "${params.assetId || ''}" to inspect the asset content (use imageConfig, videoConfig, or docConfig according to the media type; for 3D assets, call read_asset directly to inspect the 4-view composite image).`,
'2. After analyzing the retrieved content, call the "autofill_metadata" tool to provide the extracted metadata values.',
'3. If "read_asset" fails or returns an error, do not autofill any metadata and report to the user directly. Do not try anything fancy, for example downloading the file or using python/ffmpeg to extract content.',
)
@@ -834,7 +843,9 @@ export async function initializeAgentSessionActivity(params: {
let position: ShumaiMediaPosition | undefined = undefined
if (c.second !== null && c.second !== undefined) {
if (proxyType === 'pdf') {
if (proxyType === '3d') {
position = { type: 'degree', degree: secondToDegree(c.second) }
} else if (proxyType === 'pdf') {
position = { type: 'page', page: Math.round(c.second) }
} else {
position = { type: 'time', seconds: c.second }
@@ -852,7 +863,14 @@ export async function initializeAgentSessionActivity(params: {
id: att.asset.id,
name: att.asset.name,
type: att.asset.type,
mediaType: attProxyType as 'image' | 'video' | 'pdf' | 'audio' | 'other' | undefined,
mediaType: attProxyType as
| 'image'
| 'video'
| 'pdf'
| 'audio'
| '3d'
| 'other'
| undefined,
mimeType: att.asset.mediaType || undefined,
})
}
@@ -350,5 +350,24 @@ describe('serializeContextToXml', () => {
'<context>\n <annotation id="test&lt;id&gt;&amp;&quot;123&quot;" />\n</context>',
)
})
it('serializes position type="degree" with integer degree attribute without seconds', () => {
const context: ShumaiMessageContext = {
currentAsset: {
id: 'ast_3d',
name: 'robot.glb',
type: 'file',
mediaType: '3d',
path: 'robot.glb',
},
position: {
type: 'degree',
degree: 45,
},
}
expect(serializeContextToXml(context)).toBe(
'<context>\n <current_asset id="ast_3d" name="robot.glb" type="file" media_type="3d" path="robot.glb" />\n <position type="degree" degree="45" />\n</context>',
)
})
})
})
@@ -4,6 +4,7 @@ import { setupTestDbHooks } from '@shumai/db/test'
import { resolveAnnotationsById } from './annotation-resolver'
import { DatabaseSessionStorage } from '../database-session-storage'
import type { SessionTreeEntry } from '@earendil-works/pi-agent-core'
import { degreeToSecond } from '@shumai/core/src/utils/model3d'
describe('resolveAnnotationsById', () => {
setupTestDbHooks()
@@ -210,6 +211,89 @@ describe('resolveAnnotationsById', () => {
])
})
it('resolves annotations and degree position timestamp from 1-on-1 chat AgentSessionEntry for 3D assets', async () => {
const user = await prisma.user.create({
data: { name: 'User 3D', email: 'user3d@test.com', password: 'pw' },
})
const team = await prisma.team.create({
data: { name: 'Team 3D' },
})
const project = await prisma.project.create({
data: { name: 'Project 3D', teamId: team.id },
})
const asset = await prisma.asset.create({
data: {
name: 'robot.glb',
type: 'file',
projectId: project.id,
status: 'uploaded',
},
})
const agentUser = await prisma.user.create({
data: { name: 'Agent 3D', email: 'agent3d@example.com', type: 'agent' },
})
const agent = await prisma.agent.create({
data: {
id: agentUser.id,
teamId: team.id,
type: 'chat',
config: { provider: 'test-provider', model: 'test-model' },
},
})
const storage = await DatabaseSessionStorage.create({
agentId: agent.id,
userId: user.id,
assetId: asset.id,
})
storage.currentMessageContext = {
currentAsset: {
id: asset.id,
name: asset.name,
type: 'file',
path: asset.name,
},
position: { type: 'degree', degree: 45 },
annotation: true,
annotations: [
{
type: 'rectangle',
x: 10,
y: 20,
width: 30,
height: 40,
},
],
}
const userEntry: SessionTreeEntry = {
type: 'message',
id: '01JTEST3DENTRY123456789',
parentId: null,
timestamp: new Date().toISOString(),
message: {
role: 'user',
content: 'Check markup at 45 degrees',
timestamp: Date.now(),
},
}
await storage.appendEntry(userEntry)
const result = await resolveAnnotationsById(asset.id, '01JTEST3DENTRY123456789')
expect(result.timestamp).toBeCloseTo(degreeToSecond(45))
expect(result.annotations).toEqual([
{
type: 'rectangle',
x: 10,
y: 20,
width: 30,
height: 40,
},
])
})
it('throws an error when AgentSessionEntry belongs to a different asset', async () => {
const user = await prisma.user.create({
data: { name: 'User', email: 'user3@test.com', password: 'pw' },
@@ -1,5 +1,6 @@
import { prisma } from '@shumai/db'
import type { ShumaiMessageContext } from '@shumai/dtos'
import { degreeToSecond } from '@shumai/core/src/utils/model3d'
export async function resolveAnnotationsById(
assetId: string,
@@ -44,7 +45,14 @@ export async function resolveAnnotationsById(
const list = details?.annotations
const annotations =
Array.isArray(list) && list.length > 0 ? (list as unknown as PrismaJson.AnnotationList) : null
const timestamp = details?.position?.type === 'time' ? details.position.seconds : null
let timestamp: number | null = null
if (details?.position?.type === 'time') {
timestamp = details.position.seconds
} else if (details?.position?.type === 'degree') {
timestamp = degreeToSecond(details.position.degree)
} else if (details?.position?.type === 'page') {
timestamp = details.position.page
}
return { annotations, timestamp }
}
+471
View File
@@ -11,6 +11,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
import * as annotationResolver from './annotation-resolver'
import { degreeToSecond } from '@shumai/core/src/utils/model3d'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
@@ -211,6 +212,476 @@ describe('readAssetTool', () => {
})
})
describe('3D Assets', () => {
it('should read 3D asset 4-views image and return image with quadrant layout description', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
storageKey: { key: 'raw/robot.glb' },
media: {
proxyType: '3d',
imageTranscodes: [{ key: 'proxy/views.webp' }],
},
} as unknown as Asset)
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from('fake-views-bytes'),
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createReadAssetTool('user-1')
const result = await tool.execute('call-3d', {
assetId: 'model-1',
annotationId: null,
imageConfig: null,
videoConfig: null,
docConfig: null,
})
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'model-1',
})
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/views.webp')
const textItem = result.content[0] as { type: 'text'; text: string }
const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
expect(textItem.text).toContain('Orthogonal 4-view composite image')
expect(textItem.text).toContain('Top-Left quadrant: Front view (0°)')
expect(textItem.text).toContain('Top-Right quadrant: Right side view (90°)')
expect(textItem.text).toContain('Bottom-Left quadrant: Back view (180°)')
expect(textItem.text).toContain('Bottom-Right quadrant: Left side view (270°)')
expect(imageItem.type).toBe('image')
expect(imageItem.data).toBe(Buffer.from('fake-views-bytes').toString('base64'))
expect(result.details.key).toBe('proxy/views.webp')
})
it('should support s3KeyOnly mode for 3D assets', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
media: {
proxyType: '3d',
imageTranscodes: [{ key: 'proxy/views.webp' }],
},
} as unknown as Asset)
const tool = createReadAssetTool('user-1')
const result = await tool.execute('call-3d-s3', {
assetId: 'model-1',
annotationId: null,
s3KeyOnly: true,
imageConfig: null,
videoConfig: null,
docConfig: null,
})
expect(result.content).toHaveLength(1)
const textItem = result.content[0] as { type: 'text'; text: string }
expect(textItem.text).toContain('proxy/views.webp')
expect(textItem.text).toContain('Front view (0°)')
expect(result.details.key).toBe('proxy/views.webp')
})
it('should throw error if videoConfig is passed for 3D assets', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
media: {
proxyType: '3d',
imageTranscodes: [{ key: 'proxy/views.webp' }],
},
} as unknown as Asset)
const tool = createReadAssetTool('user-1')
await expect(
tool.execute('call-3d-err', {
assetId: 'model-1',
annotationId: null,
imageConfig: null,
videoConfig: { start: 0, end: 4, count: 4 },
docConfig: null,
}),
).rejects.toThrow(
"To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig",
)
})
it('should throw error if 3D asset imageTranscodes is missing when overview is requested', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
media: {
proxyType: '3d',
imageTranscodes: [],
},
} as unknown as Asset)
const tool = createReadAssetTool('user-1')
await expect(
tool.execute('call-3d-err2', {
assetId: 'model-1',
annotationId: null,
imageConfig: null,
videoConfig: null,
docConfig: null,
}),
).rejects.toThrow('4-views image (views.webp) is missing')
})
it('should extract specific turntable frame when config3d.degree is provided', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
projectId: 'proj-1',
media: {
proxyType: '3d',
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
imageTranscodes: [{ key: 'proxy/views.webp' }],
},
} as unknown as Asset)
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
id: 'task-shot-3d',
} as unknown as WorkflowTask)
vi.mocked(workflowService.executeWait).mockResolvedValue({
id: 'task-shot-3d',
status: WorkflowTaskStatus.completed,
output: {
screenshots: [{ key: 'proxy/shot-45.webp', timestamp: 0.5833 }],
},
} as unknown as WorkflowTask)
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from('fake-shot-45-bytes'),
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createReadAssetTool('user-1')
const result = await tool.execute('call-3d-degree', {
assetId: 'model-1',
annotationId: null,
config3d: { degree: 45 },
imageConfig: null,
videoConfig: null,
docConfig: null,
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: expect.objectContaining({
type: WorkflowTaskType.transcode_screenshot,
assetId: 'model-1',
payload: {
projectId: 'proj-1',
screenshot: {
start: expect.any(Number),
end: expect.any(Number),
count: 1,
commentTimestamp: expect.any(Number),
annotations: undefined,
},
},
}),
})
const textItem = result.content[0] as { type: 'text'; text: string }
const imageItem = result.content[1] as { type: 'image'; data: string; mimeType: string }
expect(textItem.text).toContain('view at 45°:')
expect(imageItem.data).toBe(Buffer.from('fake-shot-45-bytes').toString('base64'))
expect(result.details.key).toBe('proxy/shot-45.webp')
})
it('should throw error when config3d.degree is not a multiple of 15 or out of range', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
media: {
proxyType: '3d',
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
},
} as unknown as Asset)
const tool = createReadAssetTool('user-1')
await expect(
tool.execute('call-3d-invalid-deg', {
assetId: 'model-1',
annotationId: null,
config3d: { degree: 40 },
imageConfig: null,
videoConfig: null,
docConfig: null,
}),
).rejects.toThrow('Invalid turntable degree: 40')
await expect(
tool.execute('call-3d-out-of-range', {
assetId: 'model-1',
annotationId: null,
config3d: { degree: 360 },
imageConfig: null,
videoConfig: null,
docConfig: null,
}),
).rejects.toThrow('Invalid turntable degree: 360')
})
it('should extract frame with overlaid markups when annotationId is provided on 3D asset', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
projectId: 'proj-1',
media: {
proxyType: '3d',
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
},
} as unknown as Asset)
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
annotations: [
{
type: 'freehand',
color: '#ff0000',
points: [
[0, 0],
[10, 10],
],
},
],
timestamp: 0.5, // 0.5s = 45°
})
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
id: 'task-shot-ann',
} as unknown as WorkflowTask)
vi.mocked(workflowService.executeWait).mockResolvedValue({
id: 'task-shot-ann',
status: WorkflowTaskStatus.completed,
output: {
screenshots: [{ key: 'proxy/shot-annotated.webp', timestamp: 0.5 }],
},
} as unknown as WorkflowTask)
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from('fake-shot-ann-bytes'),
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createReadAssetTool('user-1')
const result = await tool.execute('call-3d-ann', {
assetId: 'model-1',
annotationId: 'comm-1',
config3d: null,
imageConfig: null,
videoConfig: null,
docConfig: null,
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: expect.objectContaining({
type: WorkflowTaskType.transcode_screenshot,
assetId: 'model-1',
payload: {
projectId: 'proj-1',
screenshot: {
start: 0.5,
end: 0.5,
count: 1,
commentTimestamp: 0.5,
annotations: [
{
type: 'freehand',
color: '#ff0000',
points: [
[0, 0],
[10, 10],
],
},
],
},
},
}),
})
const textItem = result.content[0] as { type: 'text'; text: string }
expect(textItem.text).toContain('view at 45° with visual markups overlaid:')
expect(result.details.key).toBe('proxy/shot-annotated.webp')
})
it('should extract turntable frame at resolved degree angle for 3D session annotations', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
projectId: 'proj-1',
media: {
proxyType: '3d',
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
},
} as unknown as Asset)
const expectedTime = degreeToSecond(90)
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
annotations: [
{
type: 'box',
color: '#00ff00',
points: [
[10, 20],
[40, 60],
],
},
],
timestamp: expectedTime,
})
vi.mocked(prisma.workflowTask.create).mockResolvedValue({
id: 'task-shot-3d-session',
} as unknown as WorkflowTask)
vi.mocked(workflowService.executeWait).mockResolvedValue({
id: 'task-shot-3d-session',
status: WorkflowTaskStatus.completed,
output: {
screenshots: [{ key: 'proxy/shot-annotated-90.webp', timestamp: expectedTime }],
},
} as unknown as WorkflowTask)
vi.mocked(s3Service.getObject).mockResolvedValue({
buffer: Buffer.from('fake-shot-90-bytes'),
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createReadAssetTool('user-1')
const result = await tool.execute('call-3d-session-ann', {
assetId: 'model-1',
annotationId: 'session-entry-1',
config3d: null,
imageConfig: null,
videoConfig: null,
docConfig: null,
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: expect.objectContaining({
type: WorkflowTaskType.transcode_screenshot,
assetId: 'model-1',
payload: {
projectId: 'proj-1',
screenshot: {
start: expectedTime,
end: expectedTime,
count: 1,
commentTimestamp: expectedTime,
annotations: [
{
type: 'box',
color: '#00ff00',
points: [
[10, 20],
[40, 60],
],
},
],
},
},
}),
})
const textItem = result.content[0] as { type: 'text'; text: string }
expect(textItem.text).toContain('view at 90° with visual markups overlaid:')
expect(result.details.key).toBe('proxy/shot-annotated-90.webp')
})
it('should throw error if config3d.degree conflicts with annotation angle', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
projectId: 'proj-1',
media: {
proxyType: '3d',
videoTranscodes: [{ key: 'proxy/turntable.mp4' }],
},
} as unknown as Asset)
vi.spyOn(annotationResolver, 'resolveAnnotationsById').mockResolvedValue({
annotations: [
{
type: 'box',
color: '#ff0000',
points: [
[0, 0],
[10, 10],
],
},
],
timestamp: 0.5, // 45°
})
const tool = createReadAssetTool('user-1')
await expect(
tool.execute('call-3d-conflict', {
assetId: 'model-1',
annotationId: 'comm-1',
config3d: { degree: 90 }, // Conflicts with 45°
imageConfig: null,
videoConfig: null,
docConfig: null,
}),
).rejects.toThrow(
'Annotation "comm-1" was drawn at 45°, but degree 90° was specified. Do not pass a conflicting degree with annotationId.',
)
})
it('should throw error if 3D asset turntable video is missing when angle inspection is requested', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'model-1',
name: 'robot.glb',
mediaType: 'model/gltf-binary',
media: {
proxyType: '3d',
videoTranscodes: [],
},
} as unknown as Asset)
const tool = createReadAssetTool('user-1')
await expect(
tool.execute('call-3d-missing-turntable', {
assetId: 'model-1',
annotationId: null,
config3d: { degree: 45 },
imageConfig: null,
videoConfig: null,
docConfig: null,
}),
).rejects.toThrow('turntable video (turntable.mp4) is missing')
})
})
describe('Video Assets', () => {
it('should throw error if videoConfig is missing for video assets', async () => {
vi.mocked(authzService.hasPermission).mockResolvedValue()
+237 -3
View File
@@ -7,6 +7,12 @@ import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
import { resolveAnnotationsById } from './annotation-resolver'
import { getFileMimeType } from '@shumai/core/src/utils/file-mime'
import { is3dModel } from '@shumai/core/src/utils/mime'
import {
isValidTurntableDegree,
secondToDegree,
degreeToSecond,
} from '@shumai/core/src/utils/model3d'
export const readAssetSchema = Type.Object(
{
@@ -37,6 +43,31 @@ export const readAssetSchema = Type.Object(
),
Type.Null(),
]),
config3d: Type.Optional(
Type.Union([
Type.Object(
{
degree: Type.Optional(
Type.Union([
Type.Integer({
minimum: 0,
maximum: 345,
description:
'Turntable rotation degree (integer: 0, 15, 30, ..., 345 in 15° increments) to inspect a specific angle of a 3D model. Leave null/omitted to view the 4-view composite overview.',
}),
Type.Null(),
]),
),
},
{
additionalProperties: false,
description:
'Configuration for 3D model assets. Pass null if inspecting another asset type. Optionally pass degree to inspect a specific turntable angle, or leave null/omitted to view the 4-view composite overview.',
},
),
Type.Null(),
]),
),
videoConfig: Type.Union([
Type.Object(
{
@@ -167,12 +198,12 @@ export function createReadAssetTool(
label: 'Read Asset',
description:
'Inspects and reads the content of an asset in the workspace. ' +
'Supports visual analysis for images, video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
'Pass assetId, and provide the specific config (imageConfig, videoConfig, or docConfig) while setting unused configs to null. ' +
'Supports visual analysis for images, 3D models (overview or specific degree angle), video frame extraction, PDF page rendering, and direct text reading for text/markdown files. ' +
'Pass assetId, and provide the specific config (imageConfig, config3d, videoConfig, or docConfig) while setting unused configs to null. ' +
'Optionally provide annotationId to view visual markups drawn on the asset.',
parameters: readAssetSchema,
execute: async (_toolCallId, params) => {
const { assetId, annotationId, s3KeyOnly, videoConfig, docConfig } = params
const { assetId, annotationId, s3KeyOnly, config3d, videoConfig, docConfig } = params
if (agentType === 'autofill' && !userId) {
if (!targetAssetId) {
@@ -243,6 +274,209 @@ export function createReadAssetTool(
const filename = asset.name?.toLowerCase() || ''
const bucket = process.env.S3_BUCKET || 'shumai'
// ----------------------------------------------------------------------
// Branch 0: 3D Assets (Orthogonal 4-View Composite Image or Degree Angle)
// ----------------------------------------------------------------------
const is3d = proxyType === '3d' || is3dModel(mediaType, filename)
if (is3d) {
if (videoConfig) {
throw new Error(
`Asset ${asset.id} is a 3D model. To inspect specific angles pass 'config3d: { degree: ... }'; do not pass videoConfig.`,
)
}
if (config3d?.degree !== undefined && config3d?.degree !== null) {
if (!isValidTurntableDegree(config3d.degree)) {
throw new Error(
`Invalid turntable degree: ${config3d.degree}. Degree must be an integer multiple of 15 between 0 and 345 (e.g. 0, 15, 30, 45, ..., 345).`,
)
}
}
let annotations: PrismaJson.AnnotationList | null = null
let commentTimestamp: number | null = null
if (annotationId) {
const resolved = await resolveAnnotationsById(asset.id, annotationId)
annotations = resolved.annotations
commentTimestamp = resolved.timestamp
if (commentTimestamp !== null && commentTimestamp !== undefined) {
const commentDegree = secondToDegree(commentTimestamp)
if (
config3d?.degree !== undefined &&
config3d?.degree !== null &&
config3d.degree !== commentDegree
) {
throw new Error(
`Annotation "${annotationId}" was drawn at ${commentDegree}°, but degree ${config3d.degree}° was specified. Do not pass a conflicting degree with annotationId.`,
)
}
}
}
if ((config3d?.degree !== undefined && config3d?.degree !== null) || annotationId) {
if (!mediaInfo?.videoTranscodes || mediaInfo.videoTranscodes.length === 0) {
throw new Error(
`Asset ${asset.id} is a 3D model, but its turntable video (turntable.mp4) is missing.`,
)
}
let targetDegree: number
let targetSecond: number
if (config3d?.degree !== undefined && config3d?.degree !== null) {
targetDegree = config3d.degree
targetSecond = degreeToSecond(targetDegree)
} else {
targetSecond =
commentTimestamp !== null && commentTimestamp !== undefined ? commentTimestamp : 0
targetDegree = secondToDegree(targetSecond)
}
const task = await prisma.workflowTask.create({
data: {
assetId: asset.id,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_screenshot,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
screenshot: {
start: targetSecond,
end: targetSecond,
count: 1,
commentTimestamp: targetSecond,
annotations: annotations ?? undefined,
},
},
},
})
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- transcode screenshot output
const output = completedTask.output as any
const screenshots = output?.screenshots as
| Array<{ key: string; timestamp: number }>
| undefined
if (!screenshots || screenshots.length === 0) {
throw new Error(
`Failed to extract 3D turntable frame for asset ${asset.id} at ${targetDegree}°.`,
)
}
const shotKey = screenshots[0].key
const downloadUrl = await s3Service.presign(
bucket,
shotKey,
'GET',
true,
asset.name || undefined,
)
const content: Array<TextContent | ImageContent> = []
const hasMarkup = Boolean(annotations && annotations.length > 0)
const textDescription = hasMarkup
? `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}° with visual markups overlaid:`
: `3D asset "${asset.name}" (ID: ${asset.id}, S3 Key: "${shotKey}") view at ${targetDegree}°:`
if (s3KeyOnly) {
content.push({
type: 'text',
text: textDescription,
})
} else {
const { buffer, contentType } = await s3Service.getObject(bucket, shotKey)
const actualMimeType =
contentType && contentType !== 'application/octet-stream' ? contentType : 'image/webp'
content.push({
type: 'text',
text: textDescription,
})
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: actualMimeType,
})
}
return {
content,
details: {
assetId: asset.id,
name: asset.name,
mediaType: asset.mediaType,
key: shotKey,
downloadUrl,
sourceKeys: [shotKey],
},
}
}
if (!mediaInfo?.imageTranscodes || mediaInfo.imageTranscodes.length === 0) {
throw new Error(
`Asset ${asset.id} is a 3D model, but its 4-views image (views.webp) is missing.`,
)
}
const mediaKey = mediaInfo.imageTranscodes[0].key
if (!mediaKey) {
throw new Error(`Asset ${asset.id} is a 3D model, but its 4-views image key is missing.`)
}
const keyToUse = mediaKey
const mimeType = 'image/webp'
const downloadUrl = await s3Service.presign(
bucket,
keyToUse,
'GET',
true,
asset.name || undefined,
)
const content: Array<TextContent | ImageContent> = []
const textDescription = [
`3D asset "${asset.name}" (ID: ${asset.id}, MIME: ${mimeType}, S3 Key: "${keyToUse}"):`,
'Orthogonal 4-view composite image (2x2 grid across 360° rotation):',
'- Top-Left quadrant: Front view (0°)',
'- Top-Right quadrant: Right side view (90°)',
'- Bottom-Left quadrant: Back view (180°)',
'- Bottom-Right quadrant: Left side view (270°)',
].join('\n')
if (s3KeyOnly) {
content.push({
type: 'text',
text: textDescription,
})
} else {
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
const actualMimeType =
contentType && contentType !== 'application/octet-stream' ? contentType : mimeType
content.push({
type: 'text',
text: textDescription,
})
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: actualMimeType,
})
}
return {
content,
details: {
assetId: asset.id,
name: asset.name,
mediaType: asset.mediaType,
key: keyToUse,
downloadUrl,
sourceKeys: [keyToUse],
},
}
}
// ----------------------------------------------------------------------
// Branch 1: Image Assets
// ----------------------------------------------------------------------
@@ -129,6 +129,42 @@ describe('createReadThreadTool', () => {
expect(textContent).toContain('- [Ai Agent] (id: reply-1, has_markup: false): Agent answer')
})
it('should format 3d asset comment timestamps as degree: X°', async () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'root-3d',
message: 'Notice this angle',
creator: { name: 'Alice' },
second: 0.5,
annotation: [{ type: 'box' }],
asset: {
media: { proxyType: '3d' },
mediaType: 'model/gltf-binary',
name: 'robot.glb',
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as any)
vi.mocked(prisma.assetComment.findMany).mockResolvedValue([
{
id: 'reply-3d-1',
message: 'Looking at back side',
creator: { name: 'Bob' },
second: 2.0,
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
] as any)
const tool = createReadThreadTool()
const result = await tool.execute('call-3d', { threadId: 'root-3d' })
const textContent = (result.content[0] as { text: string }).text
expect(textContent).toContain(
'Thread Root [Alice] (id: root-3d, degree: 45°, has_markup: true): Notice this angle',
)
expect(textContent).toContain(
'- [Bob] (id: reply-3d-1, degree: 180°, has_markup: false): Looking at back side',
)
})
it('should describe threadId parameter as referencing <thread id="..." /> in <context>', () => {
const tool = createReadThreadTool()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
+26 -9
View File
@@ -1,6 +1,8 @@
import { prisma } from '@shumai/db'
import { Type } from 'typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { secondToDegree } from '@shumai/core/src/utils/model3d'
import { getProxyType } from '@shumai/core/src/utils/mime'
const readThreadSchema = Type.Object({
threadId: Type.String({
@@ -9,14 +11,21 @@ const readThreadSchema = Type.Object({
}),
})
function formatCommentMeta(c: {
id: string
second?: number | null
annotation?: unknown
}): string {
function formatCommentMeta(
c: {
id: string
second?: number | null
annotation?: unknown
},
proxyType?: string,
): string {
const parts = [`id: ${c.id}`]
if (c.second !== null && c.second !== undefined) {
parts.push(`time: ${c.second}s`)
if (proxyType === '3d') {
parts.push(`degree: ${secondToDegree(c.second)}°`)
} else {
parts.push(`time: ${c.second}s`)
}
}
const hasMarkup = Boolean(c.annotation && Array.isArray(c.annotation) && c.annotation.length > 0)
parts.push(`has_markup: ${hasMarkup}`)
@@ -34,7 +43,10 @@ export const createReadThreadTool = (): AgentTool<
execute: async (_toolCallId, params) => {
const rootComment = await prisma.assetComment.findUnique({
where: { id: params.threadId },
include: { creator: true },
include: {
creator: true,
asset: { select: { media: true, mediaType: true, name: true } },
},
})
if (!rootComment) {
@@ -61,9 +73,14 @@ export const createReadThreadTool = (): AgentTool<
include: { creator: true },
})
const proxyType =
(rootComment.asset?.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(rootComment.asset?.mediaType, rootComment.asset?.name) ||
undefined
const isRootAgent = rootComment.creator?.type === 'agent' || rootComment.sessionId !== null
const rootAuthor = rootComment.creator?.name || (isRootAgent ? 'Ai Agent' : 'User')
const rootMeta = formatCommentMeta(rootComment)
const rootMeta = formatCommentMeta(rootComment, proxyType)
let output = `Thread Root [${rootAuthor}] ${rootMeta}: ${rootComment.message || ''}\n\nReplies (${replies.length}):`
if (replies.length === 0) {
@@ -72,7 +89,7 @@ export const createReadThreadTool = (): AgentTool<
for (const reply of replies) {
const isAgent = reply.creator?.type === 'agent' || reply.sessionId !== null
const author = reply.creator?.name || (isAgent ? 'Ai Agent' : 'User')
const replyMeta = formatCommentMeta(reply)
const replyMeta = formatCommentMeta(reply, proxyType)
output += `\n- [${author}] ${replyMeta}: ${reply.message || ''}`
}
}
@@ -806,4 +806,89 @@ describe('Agent Chat Workflow', () => {
}),
)
})
it('should format mediaPosition as degree for 3D assets in 1-on-1 chat', async () => {
mockActivities.getAssetActivity.mockResolvedValue({
id: 'a-3d',
project: { teamId: 't1' },
type: 'file',
name: 'model.glb',
mediaType: 'model/gltf-binary',
media: { proxyType: '3d' },
parentId: null,
})
const task = await prisma.workflowTask.create({
data: {
type: 'chat',
status: 'pending',
assetId: 'a-3d',
payload: {
projectId: 'p1',
agent: {
agentId: 'b1',
prompt: 'Look at this angle',
sessionId: 'session-3d-1',
second: 0.5, // 0.5s turntable time = 45°
},
},
},
})
await agentChat(task)
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
expect.objectContaining({
messageContext: expect.objectContaining({
position: { type: 'degree', degree: 45 },
}),
}),
)
})
it('should format mediaPosition as degree for 3D assets in comment mode', async () => {
mockActivities.getAssetActivity.mockResolvedValue({
id: 'a-3d',
project: { teamId: 't1' },
type: 'file',
name: 'model.glb',
mediaType: 'model/gltf-binary',
media: { proxyType: '3d' },
parentId: null,
})
mockActivities.getCommentActivity.mockResolvedValue({
id: 'comment-3d-1',
creatorId: 'user-c1',
message: 'Look at the back',
replyToId: null,
second: 2.0, // 2.0s = 180°
attachments: [],
})
const task = await prisma.workflowTask.create({
data: {
type: 'chat',
status: 'pending',
assetId: 'a-3d',
payload: {
projectId: 'p1',
agent: {
agentId: 'b1',
userCommentId: 'comment-3d-1',
},
},
},
})
await agentChat(task)
expect(mockActivities.agentChatActivity).toHaveBeenCalledWith(
expect.objectContaining({
messageContext: expect.objectContaining({
position: { type: 'degree', degree: 180 },
}),
}),
)
})
})
+4 -1
View File
@@ -8,6 +8,7 @@ import type {
ShumaiMediaPosition,
} from '@shumai/dtos'
import { getProxyType } from '@shumai/core/src/utils/mime'
import { secondToDegree } from '@shumai/core/src/utils/model3d'
export async function agentChat(task: WorkflowTask): Promise<void> {
const {
@@ -273,7 +274,9 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
// Build position
let mediaPosition: ShumaiMediaPosition | undefined
if (commentTimestamp !== undefined && commentTimestamp !== null) {
if (proxyType === 'pdf') {
if (proxyType === '3d') {
mediaPosition = { type: 'degree', degree: secondToDegree(commentTimestamp) }
} else if (proxyType === 'pdf') {
mediaPosition = { type: 'page', page: Math.round(commentTimestamp) }
} else {
mediaPosition = { type: 'time', seconds: commentTimestamp }
@@ -211,6 +211,68 @@ describe('Agent Embedding Workflow', () => {
})
})
it('should run agent embedding workflow successfully for 3D asset using 4-views image', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'model-1',
mediaType: 'model/gltf-binary',
storageKey: { key: 'raw/model.glb' },
media: {
proxyType: '3d',
imageTranscodes: [{ key: 'files/views.webp', format: 'webp' }],
},
},
chunkDuration: 60.0,
chunkOverlap: 5.0,
})
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'model-1',
teamId: 't1',
},
})
await agentEmbeddingMedia(task)
expect(mockActivities.generateImageEmbeddingActivity).toHaveBeenCalledWith({
teamId: 't1',
assetKey: 'files/views.webp',
mediaType: 'image/webp',
})
})
it('should reject 3D asset for embeddings if imageTranscodes is missing', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
asset: {
id: 'model-1',
mediaType: 'model/gltf-binary',
storageKey: { key: 'raw/model.glb' },
media: {
proxyType: '3d',
imageTranscodes: [],
},
},
})
const task = await prisma.workflowTask.create({
data: {
type: 'ai_embedding',
status: 'pending',
assetId: 'model-1',
teamId: 't1',
},
})
await expect(agentEmbeddingMedia(task)).rejects.toThrow(
'unsupported media type for embeddings: model/gltf-binary',
)
})
it('should run agent embedding workflow successfully for video in chunks', async () => {
mockActivities.getEmbeddingContextActivity.mockResolvedValue({
agent: { id: 'b1' },
@@ -70,7 +70,10 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise<void> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const { asset, chunkDuration, chunkOverlap } = context as any
const isImage = asset.mediaType?.startsWith('image/')
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
const is3dWithViews =
mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0
const isImage = asset.mediaType?.startsWith('image/') || is3dWithViews
const isVideo = asset.mediaType?.startsWith('video/')
if (!isImage && !isVideo) {
+1
View File
@@ -32,4 +32,5 @@ export * from './media-generation/media-generation'
export * from './sandbox'
export * from './asset/comments-export'
export * from './turntable/turntable'
export * from './utils/model3d'
export * from './integrations/helper'
@@ -4432,6 +4432,89 @@ describe('TranscodeService', () => {
})
})
describe('generate3dViews', () => {
it('executes ffmpeg with 2x2 grid filter, libwebp encoder, and q=85', async () => {
let executedArgs: string[] = []
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(
_cmd: string,
args: string[],
cb: (err: unknown, result: { stdout: string; stderr: string }) => void,
) => {
executedArgs = args
cb(null, { stdout: '', stderr: '' })
},
)
const inputPath = path.join(tempDir, 'turntable.mp4')
const outputPath = path.join(tempDir, 'views.webp')
await transcodeService.generate3dViews(inputPath, outputPath)
expect(child_process.execFile).toHaveBeenCalledWith(
'ffmpeg',
expect.any(Array),
expect.any(Function),
)
expect(executedArgs).toEqual([
'-y',
'-loglevel',
'warning',
'-i',
inputPath,
'-filter_complex',
"[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]",
'-map',
'[views_out]',
'-frames:v',
'1',
'-c:v',
'libwebp',
'-q:v',
'85',
outputPath,
])
})
it('passes abort signal to execFile when provided', async () => {
const controller = new AbortController()
let optionsPassed: unknown = null
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(_cmd: string, _args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
const cb = (typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb) as (
err: unknown,
result: { stdout: string; stderr: string },
) => void
if (typeof optionsOrCb !== 'function') {
optionsPassed = optionsOrCb
}
cb(null, { stdout: '', stderr: '' })
},
)
const inputPath = path.join(tempDir, 'turntable.mp4')
const outputPath = path.join(tempDir, 'views.webp')
await transcodeService.generate3dViews(inputPath, outputPath, controller.signal)
expect(optionsPassed).toMatchObject({ signal: controller.signal })
})
it('rejects with cancellation error if aborted before execution', async () => {
const controller = new AbortController()
controller.abort()
const inputPath = path.join(tempDir, 'aborted.mp4')
const outputPath = path.join(tempDir, 'views.webp')
await expect(
transcodeService.generate3dViews(inputPath, outputPath, controller.signal),
).rejects.toThrow('3D views generation cancelled')
})
})
describe('HLS utilities', () => {
it('builds master playlist correctly with bandwidth and codecs', () => {
const playlist = buildHlsMasterPlaylist([
+31
View File
@@ -3192,6 +3192,37 @@ export class TranscodeService {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
}
async generate3dViews(
inputFile: string,
outputViews: string,
signal?: AbortSignal,
): Promise<void> {
if (signal?.aborted) {
throw new Error('3D views generation cancelled')
}
const filterComplex = `[0:v]select='eq(n\\,0)+eq(n\\,6)+eq(n\\,12)+eq(n\\,18)',scale=1440:1440,tile=2x2[views_out]`
const args = [
'-i',
inputFile,
'-filter_complex',
filterComplex,
'-map',
'[views_out]',
'-frames:v',
'1',
'-c:v',
'libwebp',
'-q:v',
'85',
outputViews,
]
if (signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
}
}
function renderAnnotationsToSvg(
+90
View File
@@ -0,0 +1,90 @@
import { describe, expect, it } from 'vitest'
import {
calculateTurntableFrameCenterTime,
degreeToSecond,
degreeToTurntableFrame,
isValidTurntableDegree,
secondToDegree,
secondToTurntableFrame,
TOTAL_TURNTABLE_FRAMES,
TURNTABLE_FPS,
} from './model3d'
describe('model3d utils', () => {
describe('secondToTurntableFrame', () => {
it('converts playback seconds to frame indices (0..23)', () => {
expect(secondToTurntableFrame(0)).toBe(0)
expect(secondToTurntableFrame(0.167)).toBe(1)
expect(secondToTurntableFrame(0.5)).toBe(3)
expect(secondToTurntableFrame(2.0)).toBe(12)
expect(secondToTurntableFrame(3.9)).toBe(23)
})
it('clamps out-of-range seconds', () => {
expect(secondToTurntableFrame(-1)).toBe(0)
expect(secondToTurntableFrame(10)).toBe(TOTAL_TURNTABLE_FRAMES - 1)
})
})
describe('secondToDegree', () => {
it('converts seconds to turntable degrees (0, 15, ..., 345)', () => {
expect(secondToDegree(0)).toBe(0)
expect(secondToDegree(1 / TURNTABLE_FPS)).toBe(15)
expect(secondToDegree(0.5)).toBe(45)
expect(secondToDegree(2.0)).toBe(180)
expect(secondToDegree(3.9)).toBe(345)
})
})
describe('degreeToTurntableFrame', () => {
it('converts discrete degrees to frame indices', () => {
expect(degreeToTurntableFrame(0)).toBe(0)
expect(degreeToTurntableFrame(15)).toBe(1)
expect(degreeToTurntableFrame(45)).toBe(3)
expect(degreeToTurntableFrame(90)).toBe(6)
expect(degreeToTurntableFrame(180)).toBe(12)
expect(degreeToTurntableFrame(345)).toBe(23)
})
it('normalizes degrees outside [0, 360)', () => {
expect(degreeToTurntableFrame(360)).toBe(0)
expect(degreeToTurntableFrame(-15)).toBe(23)
expect(degreeToTurntableFrame(720)).toBe(0)
})
})
describe('calculateTurntableFrameCenterTime and degreeToSecond', () => {
it('calculates frame center times within bounds', () => {
const centerTime0 = calculateTurntableFrameCenterTime(0)
expect(centerTime0).toBeCloseTo(0.5 / TURNTABLE_FPS)
const centerTime23 = calculateTurntableFrameCenterTime(23)
expect(centerTime23).toBeLessThan(TOTAL_TURNTABLE_FRAMES / TURNTABLE_FPS)
expect(centerTime23).toBeGreaterThan(23 / TURNTABLE_FPS)
expect(degreeToSecond(0)).toBeCloseTo(centerTime0)
expect(degreeToSecond(45)).toBeCloseTo((3 + 0.5) / TURNTABLE_FPS)
expect(degreeToSecond(345)).toBeCloseTo(centerTime23)
})
})
describe('isValidTurntableDegree', () => {
it('returns true for integer multiples of 15 from 0 to 345', () => {
for (let deg = 0; deg < 360; deg += 15) {
expect(isValidTurntableDegree(deg)).toBe(true)
}
})
it('returns false for non-multiples of 15', () => {
expect(isValidTurntableDegree(10)).toBe(false)
expect(isValidTurntableDegree(44)).toBe(false)
expect(isValidTurntableDegree(45.5)).toBe(false)
})
it('returns false for out of range or negative degrees', () => {
expect(isValidTurntableDegree(-15)).toBe(false)
expect(isValidTurntableDegree(360)).toBe(false)
expect(isValidTurntableDegree(720)).toBe(false)
})
})
})
+58
View File
@@ -0,0 +1,58 @@
export const TOTAL_TURNTABLE_FRAMES = 24
export const TURNTABLE_FPS = 6
export const TURNTABLE_FRAME_DURATION = 1 / TURNTABLE_FPS
/**
* Converts a playback second into a turntable frame index (0..23), clamped.
*/
export function secondToTurntableFrame(second: number): number {
return Math.min(
Math.max(0, Math.floor(second * TURNTABLE_FPS + 1e-4)),
TOTAL_TURNTABLE_FRAMES - 1,
)
}
/**
* Converts a turntable playback second to turntable degree (0, 15, 30, ..., 345).
*/
export function secondToDegree(second: number): number {
const frameIndex = secondToTurntableFrame(second)
return (frameIndex * 15) % 360
}
/**
* Calculates the presentation time within a turntable frame.
* Seeking near frame center avoids PTS boundary inaccuracies in video decoders.
*/
export function calculateTurntableFrameCenterTime(frameIndex: number): number {
const normalizedFrame =
((frameIndex % TOTAL_TURNTABLE_FRAMES) + TOTAL_TURNTABLE_FRAMES) % TOTAL_TURNTABLE_FRAMES
if (normalizedFrame === TOTAL_TURNTABLE_FRAMES - 1) {
return normalizedFrame * TURNTABLE_FRAME_DURATION + TURNTABLE_FRAME_DURATION * 0.25
}
return (normalizedFrame + 0.5) * TURNTABLE_FRAME_DURATION
}
/**
* Converts a rotation degree (0..345) to a turntable frame index (0..23).
*/
export function degreeToTurntableFrame(degree: number): number {
const normalizedDegree = ((Math.round(degree) % 360) + 360) % 360
return Math.round(normalizedDegree / 15) % TOTAL_TURNTABLE_FRAMES
}
/**
* Converts a rotation degree into a turntable video playback second (center time of that frame).
*/
export function degreeToSecond(degree: number): number {
const frameIndex = degreeToTurntableFrame(degree)
return calculateTurntableFrameCenterTime(frameIndex)
}
/**
* Validates whether a degree is a valid discrete turntable angle:
* integer multiple of 15 between 0 and 345 inclusive (0, 15, 30, ..., 345).
*/
export function isValidTurntableDegree(degree: number): boolean {
return Number.isInteger(degree) && degree >= 0 && degree <= 345 && degree % 15 === 0
}
+3 -2
View File
@@ -47,7 +47,7 @@ export const shumaiAssetContextSchema = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(),
mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(),
mimeType: z.string().optional(),
parentId: z.string().optional(),
path: z.string().optional(),
@@ -64,7 +64,7 @@ export const shumaiAttachedFileContextSchema = z.object({
id: z.string(),
name: z.string(),
type: z.string(),
mediaType: z.enum(['image', 'video', 'pdf', 'audio', 'other']).optional(),
mediaType: z.enum(['image', 'video', 'pdf', 'audio', '3d', 'other']).optional(),
mimeType: z.string().optional(),
path: z.string().optional(),
url: z.string().optional(),
@@ -75,6 +75,7 @@ export type ShumaiAttachedFileContext = z.infer<typeof shumaiAttachedFileContext
export const shumaiMediaPositionSchema = z.discriminatedUnion('type', [
z.object({ type: z.literal('time'), seconds: z.number() }),
z.object({ type: z.literal('page'), page: z.number() }),
z.object({ type: z.literal('degree'), degree: z.number().int() }),
])
export type ShumaiMediaPosition = z.infer<typeof shumaiMediaPositionSchema>
+2
View File
@@ -104,6 +104,8 @@ export function serializeContextToXml(context?: ShumaiMessageContext | null): st
lines.push(` <position type="time" seconds="${context.position.seconds.toFixed(4)}" />`)
} else if (context.position.type === 'page') {
lines.push(` <position type="page" page="${Math.round(context.position.page)}" />`)
} else if (context.position.type === 'degree') {
lines.push(` <position type="degree" degree="${Math.round(context.position.degree)}" />`)
}
}
@@ -4,6 +4,7 @@ import {
start3dRenderActivity,
waitFor3dVideoRenderActivity,
generate3dSpriteActivity,
generate3dViewsActivity,
} from './transcode'
import { turntableService } from '@shumai/core/src/turntable/turntable'
import { s3Service } from '@shumai/core/src/s3/s3'
@@ -31,6 +32,7 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({
transcodeService: {
convertImageToWebp: vi.fn(),
generate3dSprite: vi.fn(),
generate3dViews: vi.fn(),
},
}))
@@ -55,6 +57,7 @@ describe('3D Transcode Activities', () => {
vi.mocked(fs.readFileSync).mockReturnValue(Buffer.from('mock-bytes'))
vi.mocked(transcodeService.convertImageToWebp).mockResolvedValue(Buffer.from('webp-bytes'))
vi.mocked(transcodeService.generate3dSprite).mockResolvedValue()
vi.mocked(transcodeService.generate3dViews).mockResolvedValue()
vi.mocked(s3Service.putObject).mockResolvedValue({} as never)
vi.mocked(turntableService.deleteTask).mockResolvedValue(true)
})
@@ -324,4 +327,29 @@ describe('3D Transcode Activities', () => {
expect(res.spriteKey).toBe('files/asset-1/sprite.webp')
})
})
describe('generate3dViewsActivity', () => {
it('generates 3d 4-views composite image and uploads to S3', async () => {
const res = await generate3dViewsActivity({
taskId: 'wf-task-4',
assetKey: 'files/asset-1/model.glb',
videoFilePath: '/tmp/turntable.mp4',
viewsKey: 'files/asset-1/views.webp',
})
expect(transcodeService.generate3dViews).toHaveBeenCalledWith(
'/tmp/turntable.mp4',
'/tmp/views.webp',
undefined,
)
expect(s3Service.putObject).toHaveBeenCalledWith(
'shumai',
'files/asset-1/views.webp',
expect.any(Buffer),
expect.any(Number),
'image/webp',
)
expect(res.viewsKey).toBe('files/asset-1/views.webp')
})
})
})
@@ -27,6 +27,7 @@ import {
transcodeVideoChunkActivity,
deleteS3ObjectActivity,
createAutofillTaskIfEnabledActivity,
createEmbeddingTaskIfEnabledActivity,
} from './transcode'
vi.mock('@shumai/core/src/s3/s3', () => ({
@@ -1717,4 +1718,115 @@ describe('Transcode Activities', () => {
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
})
})
describe('createEmbeddingTaskIfEnabledActivity', () => {
it('should create ai_embedding task for 3D asset with imageTranscodes', async () => {
const user = await prisma.user.create({
data: { name: 'Test User Embedding', email: 'test-embedding-agent@test.com' },
})
const team = await prisma.team.create({
data: { name: 'Test Team Embedding' },
})
await prisma.teamMember.create({
data: { teamId: team.id, userId: user.id, role: 'owner' },
})
const project = await prisma.project.create({
data: { name: 'Test Project Embedding', teamId: team.id },
})
const embeddingAgent = await prisma.agent.create({
data: {
id: user.id,
type: 'embedding',
enabled: true,
teamId: team.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
config: { provider: 'test', model: 'test' } as any,
},
})
const asset = await prisma.asset.create({
data: {
name: 'robot.glb',
mediaType: 'model/gltf-binary',
type: 'file',
status: 'processed',
projectId: project.id,
media: {
proxyType: '3d',
imageTranscodes: [
{
key: 'files/asset-1/views.webp',
width: 2880,
height: 2880,
quality: 85,
format: 'webp',
},
],
} as unknown as PrismaJson.MediaInfo,
},
})
await createEmbeddingTaskIfEnabledActivity({
assetId: asset.id,
teamId: team.id,
projectId: project.id,
})
const task = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: 'ai_embedding' },
})
expect(task).toBeDefined()
expect(task?.status).toBe('pending')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((task?.payload as any)?.agent?.agentId).toBe(embeddingAgent.id)
})
it('should NOT create ai_embedding task for 3D asset without imageTranscodes', async () => {
const user = await prisma.user.create({
data: { name: 'Test User Embedding 2', email: 'test-embedding-agent-2@test.com' },
})
const team = await prisma.team.create({
data: { name: 'Test Team Embedding 2' },
})
await prisma.teamMember.create({
data: { teamId: team.id, userId: user.id, role: 'owner' },
})
const project = await prisma.project.create({
data: { name: 'Test Project Embedding 2', teamId: team.id },
})
await prisma.agent.create({
data: {
id: user.id,
type: 'embedding',
enabled: true,
teamId: team.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
config: { provider: 'test', model: 'test' } as any,
},
})
const asset = await prisma.asset.create({
data: {
name: 'robot.glb',
mediaType: 'model/gltf-binary',
type: 'file',
status: 'processed',
projectId: project.id,
media: {
proxyType: '3d',
imageTranscodes: [],
} as unknown as PrismaJson.MediaInfo,
},
})
await createEmbeddingTaskIfEnabledActivity({
assetId: asset.id,
teamId: team.id,
projectId: project.id,
})
const task = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: 'ai_embedding' },
})
expect(task).toBeNull()
})
})
})
+52 -1
View File
@@ -1360,8 +1360,11 @@ export async function createEmbeddingTaskIfEnabledActivity(
return
}
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
const isVideo = asset.mediaType?.startsWith('video/')
const isImage = asset.mediaType?.startsWith('image/')
const isImage =
asset.mediaType?.startsWith('image/') ||
(mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0)
if (!isVideo && !isImage) {
return
}
@@ -1886,3 +1889,51 @@ export async function generate3dSpriteActivity(
})
}
}
export interface Generate3dViewsActivityParams {
taskId: string
assetKey: string
videoFilePath: string
viewsKey: string
}
export async function generate3dViewsActivity(
params: Generate3dViewsActivityParams,
): Promise<{ viewsKey: string }> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: '3D views generation cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = path.dirname(params.videoFilePath)
const viewsFilePath = path.join(tmpDir, 'views.webp')
try {
await transcodeService.generate3dViews(params.videoFilePath, viewsFilePath, signal)
const viewsBuffer = fs.readFileSync(viewsFilePath)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.viewsKey,
viewsBuffer,
viewsBuffer.length,
'image/webp',
)
return { viewsKey: params.viewsKey }
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
const errMsg = err instanceof Error ? err.message : String(err)
throw ApplicationFailure.create({
message: `Failed to generate 3D views: ${errMsg}`,
nonRetryable: true,
})
}
}
@@ -29,6 +29,7 @@ describe('transcode3dWorkflow', () => {
_activityName: 'waitFor3dVideoRenderActivity',
}),
generate3dSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generate3dSpriteActivity' }),
generate3dViewsActivity: Object.assign(vi.fn(), { _activityName: 'generate3dViewsActivity' }),
updateAssetMediaActivity: Object.assign(vi.fn(), {
_activityName: 'updateAssetMediaActivity',
}),
@@ -173,6 +174,13 @@ describe('transcode3dWorkflow', () => {
spriteKey: 'files/asset-3d/sprite.webp',
})
expect(mockActivities.generate3dViewsActivity).toHaveBeenCalledWith({
taskId: 'task-3d',
assetKey: 'files/asset-3d/robot.glb',
videoFilePath: '/tmp/turntable.mp4',
viewsKey: 'files/asset-3d/views.webp',
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith(
expect.objectContaining({
assetId: 'asset-3d',
@@ -182,6 +190,15 @@ describe('transcode3dWorkflow', () => {
frames: 24,
poster: { key: 'files/asset-3d/poster.webp' },
sprite: expect.objectContaining({ key: 'files/asset-3d/sprite.webp' }),
imageTranscodes: [
{
key: 'files/asset-3d/views.webp',
width: 2880,
height: 2880,
quality: 85,
format: 'webp',
},
],
}),
}),
)
@@ -23,6 +23,7 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise<void> {
start3dRenderActivity,
waitFor3dVideoRenderActivity,
generate3dSpriteActivity,
generate3dViewsActivity,
updateAssetMediaActivity,
createEmbeddingTaskIfEnabledActivity,
createAutofillTaskIfEnabledActivity,
@@ -46,6 +47,7 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise<void> {
const videoKey = assetDir ? `${assetDir}/turntable.mp4` : 'turntable.mp4'
const posterKey = assetDir ? `${assetDir}/poster.webp` : 'poster.webp'
const spriteKey = assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp'
const viewsKey = assetDir ? `${assetDir}/views.webp` : 'views.webp'
const renderStart = await executeActivity(workerQueue, start3dRenderActivity, {
taskId: task.id,
@@ -117,6 +119,23 @@ export async function transcode3dWorkflow(task: WorkflowTask): Promise<void> {
tileY: 10,
}
await executeActivity(workerQueue, generate3dViewsActivity, {
taskId: task.id,
assetKey: key,
videoFilePath: renderVideo.videoFilePath,
viewsKey,
})
mediaInfo.imageTranscodes = [
{
key: viewsKey,
width: 2880,
height: 2880,
quality: 85,
format: 'webp',
},
]
await executeActivity(workerQueue, updateAssetMediaActivity, {
assetId: asset.id,
mediaInfo,