mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 15:57:14 +00:00
fix(agent): update agent tools to throw errors directly on failure
This commit is contained in:
@@ -11,6 +11,7 @@ data/
|
||||
*.db
|
||||
*.diff
|
||||
*.orig
|
||||
.agents
|
||||
|
||||
# Playwright
|
||||
/playwright-report/
|
||||
|
||||
@@ -23,126 +23,103 @@ export function createAnalyzeImageTool(
|
||||
'Retrieves and views the image content. Call this tool if you need to analyze the image.',
|
||||
parameters: analyzeImageSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const assetId = params.assetId
|
||||
const assetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
|
||||
details: {},
|
||||
}
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
// Get default media key
|
||||
let mediaKey = asset.storageKey?.key
|
||||
if (asset.media) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const mediaInfo = asset.media as any
|
||||
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
|
||||
}
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
if (!mediaKey) {
|
||||
throw new Error('No media content found for this asset.')
|
||||
}
|
||||
|
||||
// Check if trigger comment has draw annotations for this asset
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset with ID ${assetId} not found.` }],
|
||||
details: {},
|
||||
if (comment && comment.assetId === assetId && comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get default media key
|
||||
let mediaKey = asset.storageKey?.key
|
||||
if (asset.media) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const mediaInfo = asset.media as any
|
||||
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
|
||||
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
|
||||
}
|
||||
}
|
||||
let keyToUse = mediaKey
|
||||
|
||||
if (!mediaKey) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'No media content found for this asset.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
// Check if trigger comment has draw annotations for this asset
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === assetId && comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let keyToUse = mediaKey
|
||||
|
||||
if (annotations) {
|
||||
// Trigger transcode workflow to draw annotation
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId,
|
||||
if (annotations) {
|
||||
// Trigger transcode workflow to draw annotation
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_image_annotation,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
imageAnnotation: {
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
imageAnnotation: {
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
if (output?.key) {
|
||||
keyToUse = output.key
|
||||
} else {
|
||||
throw new Error('Image annotation transcode workflow failed to output S3 key.')
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve the image content from S3
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
|
||||
|
||||
const content: ImageContent[] = [
|
||||
{
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys: [keyToUse],
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error analyzing image: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
if (output?.key) {
|
||||
keyToUse = output.key
|
||||
} else {
|
||||
throw new Error('Image annotation transcode workflow failed to output S3 key.')
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve the image content from S3
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
|
||||
|
||||
const content: ImageContent[] = [
|
||||
{
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys: [keyToUse],
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,45 +58,33 @@ export function createCreateFileTool(userId: string): AgentTool<typeof createFil
|
||||
description: 'Create a new file in a specified parent folder from a local file path.',
|
||||
parameters: createFileSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const absolutePath = path.resolve(process.cwd(), params.path)
|
||||
if (!fs.existsSync(absolutePath)) {
|
||||
throw new Error(`Local file not found at path: ${params.path}`)
|
||||
}
|
||||
const absolutePath = path.resolve(process.cwd(), params.path)
|
||||
if (!fs.existsSync(absolutePath)) {
|
||||
throw new Error(`Local file not found at path: ${params.path}`)
|
||||
}
|
||||
|
||||
const fileSize = fs.statSync(absolutePath).size
|
||||
const mimeType = getMimeType(absolutePath)
|
||||
const fileSize = fs.statSync(absolutePath).size
|
||||
const mimeType = getMimeType(absolutePath)
|
||||
|
||||
// Generate compliant S3 key matching normal file upload format
|
||||
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
|
||||
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
|
||||
// Generate compliant S3 key matching normal file upload format
|
||||
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
|
||||
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
|
||||
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'create_file',
|
||||
args: {
|
||||
parent: params.parent,
|
||||
s3Key,
|
||||
name: path.basename(absolutePath),
|
||||
size: fileSize,
|
||||
contentType: mimeType,
|
||||
},
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error creating file: ${error instanceof Error ? error.message : String(error)}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'create_file',
|
||||
args: {
|
||||
parent: params.parent,
|
||||
s3Key,
|
||||
name: path.basename(absolutePath),
|
||||
size: fileSize,
|
||||
contentType: mimeType,
|
||||
},
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -19,27 +19,15 @@ export function createCreateFolderTool(userId: string): AgentTool<typeof createF
|
||||
description: 'Create a new folder in a specified parent folder.',
|
||||
parameters: createFolderSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'create_folder',
|
||||
args: params,
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error creating folder: ${error instanceof Error ? error.message : String(error)}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'create_folder',
|
||||
args: params,
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -59,45 +59,33 @@ export function createCreateVersionTool(userId: string): AgentTool<typeof create
|
||||
description: 'Create a new version of an existing file from a local file path.',
|
||||
parameters: createVersionSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const absolutePath = path.resolve(process.cwd(), params.path)
|
||||
if (!fs.existsSync(absolutePath)) {
|
||||
throw new Error(`Local file not found at path: ${params.path}`)
|
||||
}
|
||||
const absolutePath = path.resolve(process.cwd(), params.path)
|
||||
if (!fs.existsSync(absolutePath)) {
|
||||
throw new Error(`Local file not found at path: ${params.path}`)
|
||||
}
|
||||
|
||||
const fileSize = fs.statSync(absolutePath).size
|
||||
const mimeType = getMimeType(absolutePath)
|
||||
const fileSize = fs.statSync(absolutePath).size
|
||||
const mimeType = getMimeType(absolutePath)
|
||||
|
||||
// Generate compliant S3 key matching normal file upload format
|
||||
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
|
||||
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
|
||||
// Generate compliant S3 key matching normal file upload format
|
||||
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
|
||||
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
|
||||
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'create_version',
|
||||
args: {
|
||||
parent: params.parent,
|
||||
s3Key,
|
||||
name: path.basename(absolutePath),
|
||||
size: fileSize,
|
||||
contentType: mimeType,
|
||||
},
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error creating file version: ${error instanceof Error ? error.message : String(error)}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'create_version',
|
||||
args: {
|
||||
parent: params.parent,
|
||||
s3Key,
|
||||
name: path.basename(absolutePath),
|
||||
size: fileSize,
|
||||
contentType: mimeType,
|
||||
},
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -34,27 +34,15 @@ export function createListAssetsTool(userId: string): AgentTool<typeof listAsset
|
||||
description: 'List the assets (files and folders) inside a parent folder with pagination.',
|
||||
parameters: listAssetsSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'list_assets',
|
||||
args: params,
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error listing assets: ${error instanceof Error ? error.message : String(error)}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
const result = await executeAgentToolWorkflow({
|
||||
toolName: 'list_assets',
|
||||
args: params,
|
||||
userId,
|
||||
assetId: params.parent,
|
||||
})
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
||||
details: result,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -151,49 +151,37 @@ describe('readPdfPagesTool', () => {
|
||||
expect(result.details.sourceKeys).toEqual(['pdf_pages/page_1.webp', 'pdf_pages/page_2.webp'])
|
||||
})
|
||||
|
||||
it('should return error text if start page is less than 1', async () => {
|
||||
it('should throw error if start page is less than 1', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Invalid page range: start page (0) must be at least 1.',
|
||||
})
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })).rejects.toThrow(
|
||||
'Invalid page range: start page (0) must be at least 1.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should return error text if start page is greater than end page', async () => {
|
||||
it('should throw error if start page is greater than end page', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Invalid page range: start page (5) must be less than or equal to end page (2).',
|
||||
})
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })).rejects.toThrow(
|
||||
'Invalid page range: start page (5) must be less than or equal to end page (2).',
|
||||
)
|
||||
})
|
||||
|
||||
it('should return error text if page range exceeds maximum limit of 20', async () => {
|
||||
it('should throw error if page range exceeds maximum limit of 20', async () => {
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
|
||||
})
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })).rejects.toThrow(
|
||||
'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should return error text if asset not found', async () => {
|
||||
it('should throw error if asset not found', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Asset with ID asset-1 not found.',
|
||||
})
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
|
||||
'Asset with ID asset-1 not found.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should return error text if proxyType is not pdf', async () => {
|
||||
it('should throw error if proxyType is not pdf', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
@@ -201,15 +189,12 @@ describe('readPdfPagesTool', () => {
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Asset asset-1 is not a PDF or document.',
|
||||
})
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
|
||||
'Asset asset-1 is not a PDF or document.',
|
||||
)
|
||||
})
|
||||
|
||||
it('should return error text if proxyType is null/undefined (unprocessed)', async () => {
|
||||
it('should throw error if proxyType is null/undefined (unprocessed)', async () => {
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-1',
|
||||
projectId: 'project-1',
|
||||
@@ -217,11 +202,8 @@ describe('readPdfPagesTool', () => {
|
||||
} as unknown as Asset)
|
||||
|
||||
const tool = createReadPdfPagesTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
|
||||
|
||||
expect(result.content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'Asset asset-1 is not a PDF or document.',
|
||||
})
|
||||
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
|
||||
'Asset asset-1 is not a PDF or document.',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -25,161 +25,115 @@ export function createReadPdfPagesTool(
|
||||
'Converts specific page range of a PDF document into 1080p WebP images for visual analysis. Maximum 20 pages allowed per call.',
|
||||
parameters: readPdfPagesSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
if (params.start < 1) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Invalid page range: start page (${params.start}) must be at least 1.`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
if (params.start < 1) {
|
||||
throw new Error(`Invalid page range: start page (${params.start}) must be at least 1.`)
|
||||
}
|
||||
|
||||
if (params.start > params.end) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
if (params.start > params.end) {
|
||||
throw new Error(
|
||||
`Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
|
||||
)
|
||||
}
|
||||
|
||||
const MAX_PAGE_RANGE = 20
|
||||
const requestedPages = params.end - params.start + 1
|
||||
if (requestedPages > MAX_PAGE_RANGE) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
const MAX_PAGE_RANGE = 20
|
||||
const requestedPages = params.end - params.start + 1
|
||||
if (requestedPages > MAX_PAGE_RANGE) {
|
||||
throw new Error(
|
||||
`Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
|
||||
)
|
||||
}
|
||||
|
||||
const targetAssetId = params.assetId
|
||||
const targetAssetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: targetAssetId,
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: targetAssetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: targetAssetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${targetAssetId} not found.`)
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (proxyType !== 'pdf') {
|
||||
throw new Error(`Asset ${targetAssetId} is not a PDF or document.`)
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for comment page and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: targetAssetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset with ID ${targetAssetId} not found.` }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (proxyType !== 'pdf') {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for comment page and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === targetAssetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
if (comment && comment.assetId === targetAssetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger transcode workflow to render PDF pages
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: targetAssetId,
|
||||
// Trigger transcode workflow to render PDF pages
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: targetAssetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_pdf_pages,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_pdf_pages,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
pdfPages: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
pdfPages: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
|
||||
|
||||
if (!pages || pages.length === 0) {
|
||||
throw new Error('No PDF page images were generated.')
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const pageItem of pages) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
})
|
||||
sourceKeys.push(pageItem.key)
|
||||
}
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
|
||||
|
||||
if (!pages || pages.length === 0) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'No PDF page images were generated.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const pageItem of pages) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
})
|
||||
sourceKeys.push(pageItem.key)
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error reading PDF pages: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -26,118 +26,95 @@ export function createScreenshotTool(
|
||||
'Takes screenshots for a video at specified interval. It returns the images to the agent context.',
|
||||
parameters: screenshotSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
try {
|
||||
const assetId = params.assetId
|
||||
const assetId = params.assetId
|
||||
|
||||
if (!userId) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
if (!userId) {
|
||||
throw new Error('User ID is required for authorization.')
|
||||
}
|
||||
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
await authzService.hasPermission({
|
||||
user: { id: userId } as User,
|
||||
permission: Permission.Read,
|
||||
type: ResourceType.Asset,
|
||||
id: assetId,
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Asset with ID ${assetId} not found.`)
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for timestamp and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
})
|
||||
|
||||
if (!asset) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset with ID ${assetId} not found.` }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for timestamp and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
if (userCommentId) {
|
||||
const comment = await prisma.assetComment.findUnique({
|
||||
where: { id: userCommentId },
|
||||
})
|
||||
if (comment && comment.assetId === assetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
if (comment && comment.assetId === assetId) {
|
||||
commentTimestamp = comment.second !== null ? comment.second : null
|
||||
if (comment.annotation) {
|
||||
const list = comment.annotation
|
||||
if (Array.isArray(list) && list.length > 0) {
|
||||
annotations = list
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trigger transcode workflow to take screenshots
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId,
|
||||
// Trigger transcode workflow to take screenshots
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId,
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
type: WorkflowTaskType.transcode_screenshot,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: asset.projectId || 'none',
|
||||
screenshot: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
count: params.count,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
screenshot: {
|
||||
start: params.start,
|
||||
end: params.end,
|
||||
count: params.count,
|
||||
commentTimestamp,
|
||||
annotations: annotations as PrismaJson.AnnotationList,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const screenshots = output?.screenshots as
|
||||
| Array<{ key: string; timestamp: number }>
|
||||
| undefined
|
||||
|
||||
if (!screenshots || screenshots.length === 0) {
|
||||
throw new Error('No screenshots were generated.')
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const shot of screenshots) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, shot.key)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
})
|
||||
sourceKeys.push(shot.key)
|
||||
}
|
||||
|
||||
const completedTask = await workflowService.executeWait(task)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const output = completedTask.output as any
|
||||
const screenshots = output?.screenshots as
|
||||
| Array<{ key: string; timestamp: number }>
|
||||
| undefined
|
||||
|
||||
if (!screenshots || screenshots.length === 0) {
|
||||
return {
|
||||
content: [{ type: 'text', text: 'No screenshots were generated.' }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
const content: ImageContent[] = []
|
||||
const sourceKeys: string[] = []
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
|
||||
for (const shot of screenshots) {
|
||||
const { buffer, contentType } = await s3Service.getObject(bucket, shot.key)
|
||||
content.push({
|
||||
type: 'image',
|
||||
data: buffer.toString('base64'),
|
||||
mimeType: contentType || 'image/webp',
|
||||
})
|
||||
sourceKeys.push(shot.key)
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error taking screenshots: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
},
|
||||
],
|
||||
details: {},
|
||||
}
|
||||
return {
|
||||
content,
|
||||
details: {
|
||||
sourceKeys,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user