Merge pull request #208 from shumaiOne/feat/agent-media-tools-refactor

feat(agent): refactor media tools to static injection with mandatory assetId and authz
This commit is contained in:
Yiling-J
2026-07-21 16:33:12 +08:00
committed by GitHub
8 changed files with 179 additions and 60 deletions
@@ -445,9 +445,9 @@ describe('DatabaseSessionStorage', () => {
providers: [],
})
// 4. Assert that the tools were instantiated using the new comment ID, not the original one
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(asset.id, 'new-comment-456')
expect(createScreenshotToolSpy).not.toHaveBeenCalled()
// 4. Assert that the tools were instantiated using the new comment ID and user ID
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
})
it('should use the current comment ID for media tools in subsequent turns (video)', async () => {
@@ -499,8 +499,8 @@ describe('DatabaseSessionStorage', () => {
providers: [],
})
// 4. Assert that the tools were instantiated using the new comment ID, not the original one
expect(createScreenshotToolSpy).toHaveBeenCalledWith(asset.id, 'new-comment-456')
expect(createAnalyzeImageToolSpy).not.toHaveBeenCalled()
// 4. Assert that the tools were instantiated using the new comment ID and user ID
expect(createScreenshotToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
expect(createAnalyzeImageToolSpy).toHaveBeenCalledWith(user.id, 'new-comment-456')
})
})
+6 -15
View File
@@ -16,7 +16,6 @@ import { DatabaseSessionStorage } from './database-session-storage'
import { createAnalyzeImageTool } from './tools/analyze-image'
import { createScreenshotTool } from './tools/screenshot'
import { createReadPdfPagesTool } from './tools/read-pdf-pages'
import { getSimpleMediaType } from './workflows/agent-chat-prompt-builder'
import { createCreateFileTool } from './tools/create-file'
import { createCreateFolderTool } from './tools/create-folder'
import { createCreateVersionTool } from './tools/create-version'
@@ -226,24 +225,16 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
}
const metadata = await storage.getMetadata()
const assetId = metadata.assetId
const userCommentId =
passedUserCommentId !== undefined ? passedUserCommentId : metadata.userCommentId
const mediaTools: AgentTool[] = []
if (assetId) {
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
const proxyType = (asset?.media as PrismaJson.MediaInfo | null)?.proxyType
const type = getSimpleMediaType(proxyType)
if (type === 'image') {
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
} else if (type === 'video') {
mediaTools.push(createScreenshotTool(assetId, userCommentId))
} else if (type === 'pdf') {
mediaTools.push(createReadPdfPagesTool(assetId, userCommentId))
}
if (userId) {
mediaTools.push(
createAnalyzeImageTool(userId, userCommentId),
createScreenshotTool(userId, userCommentId),
createReadPdfPagesTool(userId, userCommentId),
)
}
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs, {
+30 -5
View File
@@ -5,16 +5,21 @@ import {
WorkflowTaskStatus,
type Asset,
type AssetComment,
type User,
type WorkflowTask,
} from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
return {
...actual,
prisma: {
user: {
findUnique: vi.fn(),
},
asset: {
findUnique: vi.fn(),
},
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
},
}))
vi.mock('@shumai/core/src/authz/authz', () => ({
authzService: {
hasPermission: vi.fn(),
},
Permission: { Read: 'Read' },
ResourceType: { Asset: 'asset' },
}))
describe('analyzeImageTool', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('should fetch the image from S3 when comment has no annotations', async () => {
it('should check user permissions and fetch the image from S3 when comment has no annotations', async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -55,6 +71,7 @@ describe('analyzeImageTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
annotation: null,
} as unknown as AssetComment)
@@ -63,8 +80,15 @@ describe('analyzeImageTool', () => {
contentType: 'image/png',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createAnalyzeImageTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', {})
const tool = createAnalyzeImageTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1' })
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'asset-1',
})
expect(result.content[0].type).toBe('image')
expect((result.content[0] as unknown as { data: string }).data).toBe(
@@ -87,6 +111,7 @@ describe('analyzeImageTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
annotation: [
{
type: 'box',
@@ -114,8 +139,8 @@ describe('analyzeImageTool', () => {
contentType: 'image/webp',
} as unknown as { buffer: Buffer; contentType: string })
const tool = createAnalyzeImageTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', {})
const tool = createAnalyzeImageTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1' })
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: {
+27 -6
View File
@@ -1,14 +1,19 @@
import { Type } from '@sinclair/typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { type ImageContent } from '@earendil-works/pi-ai'
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
const analyzeImageSchema = Type.Object({})
const analyzeImageSchema = Type.Object({
assetId: Type.String({
description: 'The asset ID of the image to analyze. This parameter is required.',
}),
})
export function createAnalyzeImageTool(
assetId: string,
userId: string,
userCommentId?: string | null,
): AgentTool<typeof analyzeImageSchema> {
return {
@@ -17,8 +22,24 @@ export function createAnalyzeImageTool(
description:
'Retrieves and views the image content. Call this tool if you need to analyze the image.',
parameters: analyzeImageSchema,
execute: async () => {
execute: async (_toolCallId, params) => {
try {
const assetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
include: { storageKey: true },
@@ -48,13 +69,13 @@ export function createAnalyzeImageTool(
}
}
// Check if trigger comment has draw annotations
// Check if trigger comment has draw annotations for this asset
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment?.annotation) {
if (comment && comment.assetId === assetId && comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
+38 -14
View File
@@ -5,16 +5,21 @@ import {
WorkflowTaskStatus,
type Asset,
type AssetComment,
type User,
type WorkflowTask,
} from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
return {
...actual,
prisma: {
user: {
findUnique: vi.fn(),
},
asset: {
findUnique: vi.fn(),
},
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
},
}))
vi.mock('@shumai/core/src/authz/authz', () => ({
authzService: {
hasPermission: vi.fn(),
},
Permission: { Read: 'Read' },
ResourceType: { Asset: 'asset' },
}))
describe('readPdfPagesTool', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.mocked(authzService.hasPermission).mockResolvedValue()
})
it('should trigger pdfPages transcode workflow and return image outputs', async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -54,6 +70,7 @@ describe('readPdfPagesTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
second: 2,
annotation: [
{
@@ -89,8 +106,15 @@ describe('readPdfPagesTool', () => {
} as unknown as { buffer: Buffer; contentType: string }
})
const tool = createReadPdfPagesTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', { start: 1, end: 2 })
const tool = createReadPdfPagesTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 2 })
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'asset-1',
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: {
@@ -128,8 +152,8 @@ describe('readPdfPagesTool', () => {
})
it('should return error text if start page is less than 1', async () => {
const tool = createReadPdfPagesTool('asset-1')
const result = await tool.execute('call-1', { start: 0, end: 5 })
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })
expect(result.content[0]).toEqual({
type: 'text',
@@ -138,8 +162,8 @@ describe('readPdfPagesTool', () => {
})
it('should return error text if start page is greater than end page', async () => {
const tool = createReadPdfPagesTool('asset-1')
const result = await tool.execute('call-1', { start: 5, end: 2 })
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })
expect(result.content[0]).toEqual({
type: 'text',
@@ -148,8 +172,8 @@ describe('readPdfPagesTool', () => {
})
it('should return error text if page range exceeds maximum limit of 20', async () => {
const tool = createReadPdfPagesTool('asset-1')
const result = await tool.execute('call-1', { start: 1, end: 25 })
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })
expect(result.content[0]).toEqual({
type: 'text',
@@ -160,8 +184,8 @@ describe('readPdfPagesTool', () => {
it('should return error text if asset not found', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
const tool = createReadPdfPagesTool('asset-1')
const result = await tool.execute('call-1', { start: 1, end: 3 })
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
@@ -176,8 +200,8 @@ describe('readPdfPagesTool', () => {
media: { proxyType: 'image' },
} as unknown as Asset)
const tool = createReadPdfPagesTool('asset-1')
const result = await tool.execute('call-1', { start: 1, end: 3 })
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
@@ -192,8 +216,8 @@ describe('readPdfPagesTool', () => {
media: null,
} as unknown as Asset)
const tool = createReadPdfPagesTool('asset-1')
const result = await tool.execute('call-1', { start: 1, end: 3 })
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
+23 -9
View File
@@ -1,22 +1,21 @@
import { Type } from '@sinclair/typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { type ImageContent } from '@earendil-works/pi-ai'
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
const readPdfPagesSchema = Type.Object({
assetId: Type.String({
description: 'Asset ID of the PDF document. This parameter is required.',
}),
start: Type.Number({ description: 'Start page number (1-based index)' }),
end: Type.Number({ description: 'End page number (1-based index)' }),
assetId: Type.Optional(
Type.String({
description: 'Asset ID of the PDF document (optional, defaults to current asset)',
}),
),
})
export function createReadPdfPagesTool(
assetId: string,
userId: string,
userCommentId?: string | null,
): AgentTool<typeof readPdfPagesSchema> {
return {
@@ -65,7 +64,22 @@ export function createReadPdfPagesTool(
}
}
const targetAssetId = params.assetId || assetId
const targetAssetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: targetAssetId,
})
const asset = await prisma.asset.findUnique({
where: { id: targetAssetId },
})
@@ -92,7 +106,7 @@ export function createReadPdfPagesTool(
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment) {
if (comment && comment.assetId === targetAssetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
+26 -2
View File
@@ -5,16 +5,21 @@ import {
WorkflowTaskStatus,
type Asset,
type AssetComment,
type User,
type WorkflowTask,
} from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService } from '@shumai/core/src/authz/authz'
vi.mock('@shumai/db', async (importOriginal) => {
const actual = await importOriginal<typeof import('@shumai/db')>()
return {
...actual,
prisma: {
user: {
findUnique: vi.fn(),
},
asset: {
findUnique: vi.fn(),
},
@@ -40,12 +45,23 @@ vi.mock('@shumai/workflow-core', () => ({
},
}))
vi.mock('@shumai/core/src/authz/authz', () => ({
authzService: {
hasPermission: vi.fn(),
},
Permission: { Read: 'Read' },
ResourceType: { Asset: 'asset' },
}))
describe('screenshotTool', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('should trigger screenshot transcode workflow and return image outputs', async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ id: 'user-1' } as User)
vi.mocked(authzService.hasPermission).mockResolvedValue()
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -53,6 +69,7 @@ describe('screenshotTool', () => {
vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({
id: 'comment-1',
assetId: 'asset-1',
second: 5.0,
annotation: [
{
@@ -88,8 +105,15 @@ describe('screenshotTool', () => {
} as unknown as { buffer: Buffer; contentType: string }
})
const tool = createScreenshotTool('asset-1', 'comment-1')
const result = await tool.execute('call-1', { start: 0, end: 10, count: 2 })
const tool = createScreenshotTool('user-1', 'comment-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 10, count: 2 })
expect(authzService.hasPermission).toHaveBeenCalledWith({
user: { id: 'user-1' },
permission: 'Read',
type: 'asset',
id: 'asset-1',
})
expect(prisma.workflowTask.create).toHaveBeenCalledWith({
data: {
+23 -3
View File
@@ -1,18 +1,22 @@
import { Type } from '@sinclair/typebox'
import { type AgentTool } from '@earendil-works/pi-agent-core'
import { type ImageContent } from '@earendil-works/pi-ai'
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { workflowService } from '@shumai/workflow-core'
import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz'
const screenshotSchema = Type.Object({
assetId: Type.String({
description: 'The asset ID of the video asset. This parameter is required.',
}),
start: Type.Number({ description: 'Start time in seconds' }),
end: Type.Number({ description: 'End time in seconds' }),
count: Type.Number({ description: 'Number of screenshots to take' }),
})
export function createScreenshotTool(
assetId: string,
userId: string,
userCommentId?: string | null,
): AgentTool<typeof screenshotSchema> {
return {
@@ -23,6 +27,22 @@ export function createScreenshotTool(
parameters: screenshotSchema,
execute: async (_toolCallId, params) => {
try {
const assetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
@@ -41,7 +61,7 @@ export function createScreenshotTool(
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment) {
if (comment && comment.assetId === assetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation