fix(agent): update agent tools to throw errors directly on failure

This commit is contained in:
yiling
2026-07-22 11:05:19 +08:00
parent bbe1e7128d
commit bc63a95bf5
9 changed files with 344 additions and 501 deletions
+1
View File
@@ -11,6 +11,7 @@ data/
*.db
*.diff
*.orig
.agents
# Playwright
/playwright-report/
+82 -105
View File
@@ -23,126 +23,103 @@ export function createAnalyzeImageTool(
'Retrieves and views the image content. Call this tool if you need to analyze the image.',
parameters: analyzeImageSchema,
execute: async (_toolCallId, params) => {
try {
const assetId = params.assetId
const assetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
if (!userId) {
throw new Error('User ID is required for authorization.')
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
include: { storageKey: true },
})
if (!asset) {
throw new Error(`Asset with ID ${assetId} not found.`)
}
// Get default media key
let mediaKey = asset.storageKey?.key
if (asset.media) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const mediaInfo = asset.media as any
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
}
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
if (!mediaKey) {
throw new Error('No media content found for this asset.')
}
// Check if trigger comment has draw annotations for this asset
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
include: { storageKey: true },
})
if (!asset) {
return {
content: [{ type: 'text', text: `Asset with ID ${assetId} not found.` }],
details: {},
if (comment && comment.assetId === assetId && comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
}
}
// Get default media key
let mediaKey = asset.storageKey?.key
if (asset.media) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const mediaInfo = asset.media as any
if (mediaInfo.imageTranscodes && mediaInfo.imageTranscodes.length > 0) {
mediaKey = mediaInfo.imageTranscodes[0].key || mediaKey
}
}
let keyToUse = mediaKey
if (!mediaKey) {
return {
content: [{ type: 'text', text: 'No media content found for this asset.' }],
details: {},
}
}
// Check if trigger comment has draw annotations for this asset
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment && comment.assetId === assetId && comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
}
}
let keyToUse = mediaKey
if (annotations) {
// Trigger transcode workflow to draw annotation
const task = await prisma.workflowTask.create({
data: {
assetId,
if (annotations) {
// Trigger transcode workflow to draw annotation
const task = await prisma.workflowTask.create({
data: {
assetId,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_image_annotation,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_image_annotation,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
imageAnnotation: {
annotations: annotations as PrismaJson.AnnotationList,
},
imageAnnotation: {
annotations: annotations as PrismaJson.AnnotationList,
},
},
})
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
if (output?.key) {
keyToUse = output.key
} else {
throw new Error('Image annotation transcode workflow failed to output S3 key.')
}
}
// Retrieve the image content from S3
const bucket = process.env.S3_BUCKET || 'shumai'
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
const content: ImageContent[] = [
{
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
},
]
})
return {
content,
details: {
sourceKeys: [keyToUse],
},
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error analyzing image: ${
error instanceof Error ? error.message : String(error)
}`,
},
],
details: {},
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
if (output?.key) {
keyToUse = output.key
} else {
throw new Error('Image annotation transcode workflow failed to output S3 key.')
}
}
// Retrieve the image content from S3
const bucket = process.env.S3_BUCKET || 'shumai'
const { buffer, contentType } = await s3Service.getObject(bucket, keyToUse)
const content: ImageContent[] = [
{
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
},
]
return {
content,
details: {
sourceKeys: [keyToUse],
},
}
},
}
}
+24 -36
View File
@@ -58,45 +58,33 @@ export function createCreateFileTool(userId: string): AgentTool<typeof createFil
description: 'Create a new file in a specified parent folder from a local file path.',
parameters: createFileSchema,
execute: async (_toolCallId, params) => {
try {
const absolutePath = path.resolve(process.cwd(), params.path)
if (!fs.existsSync(absolutePath)) {
throw new Error(`Local file not found at path: ${params.path}`)
}
const absolutePath = path.resolve(process.cwd(), params.path)
if (!fs.existsSync(absolutePath)) {
throw new Error(`Local file not found at path: ${params.path}`)
}
const fileSize = fs.statSync(absolutePath).size
const mimeType = getMimeType(absolutePath)
const fileSize = fs.statSync(absolutePath).size
const mimeType = getMimeType(absolutePath)
// Generate compliant S3 key matching normal file upload format
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
// Generate compliant S3 key matching normal file upload format
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
const result = await executeAgentToolWorkflow({
toolName: 'create_file',
args: {
parent: params.parent,
s3Key,
name: path.basename(absolutePath),
size: fileSize,
contentType: mimeType,
},
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error creating file: ${error instanceof Error ? error.message : String(error)}`,
},
],
details: {},
}
const result = await executeAgentToolWorkflow({
toolName: 'create_file',
args: {
parent: params.parent,
s3Key,
name: path.basename(absolutePath),
size: fileSize,
contentType: mimeType,
},
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
},
}
+9 -21
View File
@@ -19,27 +19,15 @@ export function createCreateFolderTool(userId: string): AgentTool<typeof createF
description: 'Create a new folder in a specified parent folder.',
parameters: createFolderSchema,
execute: async (_toolCallId, params) => {
try {
const result = await executeAgentToolWorkflow({
toolName: 'create_folder',
args: params,
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error creating folder: ${error instanceof Error ? error.message : String(error)}`,
},
],
details: {},
}
const result = await executeAgentToolWorkflow({
toolName: 'create_folder',
args: params,
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
},
}
+24 -36
View File
@@ -59,45 +59,33 @@ export function createCreateVersionTool(userId: string): AgentTool<typeof create
description: 'Create a new version of an existing file from a local file path.',
parameters: createVersionSchema,
execute: async (_toolCallId, params) => {
try {
const absolutePath = path.resolve(process.cwd(), params.path)
if (!fs.existsSync(absolutePath)) {
throw new Error(`Local file not found at path: ${params.path}`)
}
const absolutePath = path.resolve(process.cwd(), params.path)
if (!fs.existsSync(absolutePath)) {
throw new Error(`Local file not found at path: ${params.path}`)
}
const fileSize = fs.statSync(absolutePath).size
const mimeType = getMimeType(absolutePath)
const fileSize = fs.statSync(absolutePath).size
const mimeType = getMimeType(absolutePath)
// Generate compliant S3 key matching normal file upload format
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
// Generate compliant S3 key matching normal file upload format
const s3Key = `files/${ulid()}/${sanitizeFilename(path.basename(absolutePath))}`
await s3Service.uploadFileToKey(absolutePath, s3Key, mimeType)
const result = await executeAgentToolWorkflow({
toolName: 'create_version',
args: {
parent: params.parent,
s3Key,
name: path.basename(absolutePath),
size: fileSize,
contentType: mimeType,
},
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error creating file version: ${error instanceof Error ? error.message : String(error)}`,
},
],
details: {},
}
const result = await executeAgentToolWorkflow({
toolName: 'create_version',
args: {
parent: params.parent,
s3Key,
name: path.basename(absolutePath),
size: fileSize,
contentType: mimeType,
},
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
},
}
+9 -21
View File
@@ -34,27 +34,15 @@ export function createListAssetsTool(userId: string): AgentTool<typeof listAsset
description: 'List the assets (files and folders) inside a parent folder with pagination.',
parameters: listAssetsSchema,
execute: async (_toolCallId, params) => {
try {
const result = await executeAgentToolWorkflow({
toolName: 'list_assets',
args: params,
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error listing assets: ${error instanceof Error ? error.message : String(error)}`,
},
],
details: {},
}
const result = await executeAgentToolWorkflow({
toolName: 'list_assets',
args: params,
userId,
assetId: params.parent,
})
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
details: result,
}
},
}
+24 -42
View File
@@ -151,49 +151,37 @@ describe('readPdfPagesTool', () => {
expect(result.details.sourceKeys).toEqual(['pdf_pages/page_1.webp', 'pdf_pages/page_2.webp'])
})
it('should return error text if start page is less than 1', async () => {
it('should throw error if start page is less than 1', async () => {
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Invalid page range: start page (0) must be at least 1.',
})
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 0, end: 5 })).rejects.toThrow(
'Invalid page range: start page (0) must be at least 1.',
)
})
it('should return error text if start page is greater than end page', async () => {
it('should throw error if start page is greater than end page', async () => {
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Invalid page range: start page (5) must be less than or equal to end page (2).',
})
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 5, end: 2 })).rejects.toThrow(
'Invalid page range: start page (5) must be less than or equal to end page (2).',
)
})
it('should return error text if page range exceeds maximum limit of 20', async () => {
it('should throw error if page range exceeds maximum limit of 20', async () => {
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
})
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 25 })).rejects.toThrow(
'Page range (25 pages requested) exceeds the maximum limit of 20 pages per request.',
)
})
it('should return error text if asset not found', async () => {
it('should throw error if asset not found', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue(null)
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Asset with ID asset-1 not found.',
})
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
'Asset with ID asset-1 not found.',
)
})
it('should return error text if proxyType is not pdf', async () => {
it('should throw error if proxyType is not pdf', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -201,15 +189,12 @@ describe('readPdfPagesTool', () => {
} as unknown as Asset)
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Asset asset-1 is not a PDF or document.',
})
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
'Asset asset-1 is not a PDF or document.',
)
})
it('should return error text if proxyType is null/undefined (unprocessed)', async () => {
it('should throw error if proxyType is null/undefined (unprocessed)', async () => {
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
id: 'asset-1',
projectId: 'project-1',
@@ -217,11 +202,8 @@ describe('readPdfPagesTool', () => {
} as unknown as Asset)
const tool = createReadPdfPagesTool('user-1')
const result = await tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })
expect(result.content[0]).toEqual({
type: 'text',
text: 'Asset asset-1 is not a PDF or document.',
})
await expect(tool.execute('call-1', { assetId: 'asset-1', start: 1, end: 3 })).rejects.toThrow(
'Asset asset-1 is not a PDF or document.',
)
})
})
+94 -140
View File
@@ -25,161 +25,115 @@ export function createReadPdfPagesTool(
'Converts specific page range of a PDF document into 1080p WebP images for visual analysis. Maximum 20 pages allowed per call.',
parameters: readPdfPagesSchema,
execute: async (_toolCallId, params) => {
try {
if (params.start < 1) {
return {
content: [
{
type: 'text',
text: `Invalid page range: start page (${params.start}) must be at least 1.`,
},
],
details: {},
}
}
if (params.start < 1) {
throw new Error(`Invalid page range: start page (${params.start}) must be at least 1.`)
}
if (params.start > params.end) {
return {
content: [
{
type: 'text',
text: `Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
},
],
details: {},
}
}
if (params.start > params.end) {
throw new Error(
`Invalid page range: start page (${params.start}) must be less than or equal to end page (${params.end}).`,
)
}
const MAX_PAGE_RANGE = 20
const requestedPages = params.end - params.start + 1
if (requestedPages > MAX_PAGE_RANGE) {
return {
content: [
{
type: 'text',
text: `Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
},
],
details: {},
}
}
const MAX_PAGE_RANGE = 20
const requestedPages = params.end - params.start + 1
if (requestedPages > MAX_PAGE_RANGE) {
throw new Error(
`Page range (${requestedPages} pages requested) exceeds the maximum limit of ${MAX_PAGE_RANGE} pages per request.`,
)
}
const targetAssetId = params.assetId
const targetAssetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
if (!userId) {
throw new Error('User ID is required for authorization.')
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: targetAssetId,
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: targetAssetId,
})
const asset = await prisma.asset.findUnique({
where: { id: targetAssetId },
})
if (!asset) {
throw new Error(`Asset with ID ${targetAssetId} not found.`)
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
if (proxyType !== 'pdf') {
throw new Error(`Asset ${targetAssetId} is not a PDF or document.`)
}
// Fetch trigger comment details for comment page and annotations
let commentTimestamp: number | null = null
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
const asset = await prisma.asset.findUnique({
where: { id: targetAssetId },
})
if (!asset) {
return {
content: [{ type: 'text', text: `Asset with ID ${targetAssetId} not found.` }],
details: {},
}
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
if (proxyType !== 'pdf') {
return {
content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }],
details: {},
}
}
// Fetch trigger comment details for comment page and annotations
let commentTimestamp: number | null = null
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment && comment.assetId === targetAssetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
if (comment && comment.assetId === targetAssetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
}
}
}
// Trigger transcode workflow to render PDF pages
const task = await prisma.workflowTask.create({
data: {
assetId: targetAssetId,
// Trigger transcode workflow to render PDF pages
const task = await prisma.workflowTask.create({
data: {
assetId: targetAssetId,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_pdf_pages,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_pdf_pages,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
pdfPages: {
start: params.start,
end: params.end,
commentTimestamp,
annotations: annotations as PrismaJson.AnnotationList,
},
pdfPages: {
start: params.start,
end: params.end,
commentTimestamp,
annotations: annotations as PrismaJson.AnnotationList,
},
},
},
})
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
if (!pages || pages.length === 0) {
throw new Error('No PDF page images were generated.')
}
const content: ImageContent[] = []
const sourceKeys: string[] = []
const bucket = process.env.S3_BUCKET || 'shumai'
for (const pageItem of pages) {
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
})
sourceKeys.push(pageItem.key)
}
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
const pages = output?.pages as Array<{ key: string; page: number }> | undefined
if (!pages || pages.length === 0) {
return {
content: [{ type: 'text', text: 'No PDF page images were generated.' }],
details: {},
}
}
const content: ImageContent[] = []
const sourceKeys: string[] = []
const bucket = process.env.S3_BUCKET || 'shumai'
for (const pageItem of pages) {
const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key)
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
})
sourceKeys.push(pageItem.key)
}
return {
content,
details: {
sourceKeys,
},
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error reading PDF pages: ${
error instanceof Error ? error.message : String(error)
}`,
},
],
details: {},
}
return {
content,
details: {
sourceKeys,
},
}
},
}
+77 -100
View File
@@ -26,118 +26,95 @@ export function createScreenshotTool(
'Takes screenshots for a video at specified interval. It returns the images to the agent context.',
parameters: screenshotSchema,
execute: async (_toolCallId, params) => {
try {
const assetId = params.assetId
const assetId = params.assetId
if (!userId) {
return {
content: [{ type: 'text', text: 'User ID is required for authorization.' }],
details: {},
}
}
if (!userId) {
throw new Error('User ID is required for authorization.')
}
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
await authzService.hasPermission({
user: { id: userId } as User,
permission: Permission.Read,
type: ResourceType.Asset,
id: assetId,
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
if (!asset) {
throw new Error(`Asset with ID ${assetId} not found.`)
}
// Fetch trigger comment details for timestamp and annotations
let commentTimestamp: number | null = null
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
if (!asset) {
return {
content: [{ type: 'text', text: `Asset with ID ${assetId} not found.` }],
details: {},
}
}
// Fetch trigger comment details for timestamp and annotations
let commentTimestamp: number | null = null
let annotations: unknown = null
if (userCommentId) {
const comment = await prisma.assetComment.findUnique({
where: { id: userCommentId },
})
if (comment && comment.assetId === assetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
if (comment && comment.assetId === assetId) {
commentTimestamp = comment.second !== null ? comment.second : null
if (comment.annotation) {
const list = comment.annotation
if (Array.isArray(list) && list.length > 0) {
annotations = list
}
}
}
}
// Trigger transcode workflow to take screenshots
const task = await prisma.workflowTask.create({
data: {
assetId,
// Trigger transcode workflow to take screenshots
const task = await prisma.workflowTask.create({
data: {
assetId,
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_screenshot,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
type: WorkflowTaskType.transcode_screenshot,
status: WorkflowTaskStatus.pending,
payload: {
projectId: asset.projectId || 'none',
screenshot: {
start: params.start,
end: params.end,
count: params.count,
commentTimestamp,
annotations: annotations as PrismaJson.AnnotationList,
},
screenshot: {
start: params.start,
end: params.end,
count: params.count,
commentTimestamp,
annotations: annotations as PrismaJson.AnnotationList,
},
},
},
})
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
const screenshots = output?.screenshots as
| Array<{ key: string; timestamp: number }>
| undefined
if (!screenshots || screenshots.length === 0) {
throw new Error('No screenshots were generated.')
}
const content: ImageContent[] = []
const sourceKeys: string[] = []
const bucket = process.env.S3_BUCKET || 'shumai'
for (const shot of screenshots) {
const { buffer, contentType } = await s3Service.getObject(bucket, shot.key)
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
})
sourceKeys.push(shot.key)
}
const completedTask = await workflowService.executeWait(task)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const output = completedTask.output as any
const screenshots = output?.screenshots as
| Array<{ key: string; timestamp: number }>
| undefined
if (!screenshots || screenshots.length === 0) {
return {
content: [{ type: 'text', text: 'No screenshots were generated.' }],
details: {},
}
}
const content: ImageContent[] = []
const sourceKeys: string[] = []
const bucket = process.env.S3_BUCKET || 'shumai'
for (const shot of screenshots) {
const { buffer, contentType } = await s3Service.getObject(bucket, shot.key)
content.push({
type: 'image',
data: buffer.toString('base64'),
mimeType: contentType || 'image/webp',
})
sourceKeys.push(shot.key)
}
return {
content,
details: {
sourceKeys,
},
}
} catch (error) {
return {
content: [
{
type: 'text',
text: `Error taking screenshots: ${
error instanceof Error ? error.message : String(error)
}`,
},
],
details: {},
}
return {
content,
details: {
sourceKeys,
},
}
},
}