From f68ba706562fadeae5937c6627beb3598fde0017 Mon Sep 17 00:00:00 2001 From: yiling Date: Sun, 19 Jul 2026 09:45:14 +0800 Subject: [PATCH] feat(agent): add read_pdf_pages tool and PDF page context support --- packages/agent/src/index.ts | 3 + .../agent/src/tools/read-pdf-pages.test.ts | 140 ++++++++++++++++++ packages/agent/src/tools/read-pdf-pages.ts | 126 ++++++++++++++++ .../agent-chat-prompt-builder.test.ts | 37 +++++ .../workflows/agent-chat-prompt-builder.ts | 32 +++- packages/agent/src/workflows/agent-chat.ts | 3 +- packages/core/src/transcode/transcode.test.ts | 50 ++++++- packages/core/src/transcode/transcode.ts | 106 +++++++++++++ packages/db/src/prisma-json-types.ts | 8 + .../src/activities/transcode.test.ts | 27 ++++ .../transcode/src/activities/transcode.ts | 30 ++++ .../transcode/src/workflows/transcode.test.ts | 65 ++++++++ packages/transcode/src/workflows/transcode.ts | 29 ++++ .../components/settings/AgentFormDialog.tsx | 5 + packages/webui/messages/en.json | 2 + packages/webui/messages/zh.json | 2 + packages/webui/paraglide/messages/_index.d.ts | 2 + packages/webui/paraglide/messages/_index.js | 2 + .../agent_tool_read_pdf_pages_desc.d.ts | 16 ++ .../agent_tool_read_pdf_pages_desc.js | 29 ++++ .../agent_tool_read_pdf_pages_name.d.ts | 16 ++ .../agent_tool_read_pdf_pages_name.js | 29 ++++ 22 files changed, 753 insertions(+), 6 deletions(-) create mode 100644 packages/agent/src/tools/read-pdf-pages.test.ts create mode 100644 packages/agent/src/tools/read-pdf-pages.ts create mode 100644 packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.d.ts create mode 100644 packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.js create mode 100644 packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.d.ts create mode 100644 packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.js diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index b3ccbe82..12c9f02f 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -15,6 +15,7 @@ import { agentService } from '@shumai/core/src/agent/agent' import { DatabaseSessionStorage } from './database-session-storage' import { createAnalyzeImageTool } from './tools/analyze-image' import { createScreenshotTool } from './tools/screenshot' +import { createReadPdfPagesTool } from './tools/read-pdf-pages' import { getSimpleMediaType } from './workflows/agent-chat-prompt-builder' import { createCreateFileTool } from './tools/create-file' import { createCreateFolderTool } from './tools/create-folder' @@ -240,6 +241,8 @@ export async function createAgentSession(params: CreateAgentSessionParams) { mediaTools.push(createAnalyzeImageTool(assetId, userCommentId)) } else if (type === 'video') { mediaTools.push(createScreenshotTool(assetId, userCommentId)) + } else if (type === 'pdf') { + mediaTools.push(createReadPdfPagesTool(assetId, userCommentId)) } } } diff --git a/packages/agent/src/tools/read-pdf-pages.test.ts b/packages/agent/src/tools/read-pdf-pages.test.ts new file mode 100644 index 00000000..97476b02 --- /dev/null +++ b/packages/agent/src/tools/read-pdf-pages.test.ts @@ -0,0 +1,140 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { createReadPdfPagesTool } from './read-pdf-pages' +import { + prisma, + WorkflowTaskStatus, + type Asset, + type AssetComment, + type WorkflowTask, +} from '@shumai/db' +import { s3Service } from '@shumai/core/src/s3/s3' +import { workflowService } from '@shumai/workflow-core' + +vi.mock('@shumai/db', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + prisma: { + asset: { + findUnique: vi.fn(), + }, + assetComment: { + findUnique: vi.fn(), + }, + workflowTask: { + create: vi.fn(), + }, + }, + } +}) + +vi.mock('@shumai/core/src/s3/s3', () => ({ + s3Service: { + getObject: vi.fn(), + }, +})) + +vi.mock('@shumai/workflow-core', () => ({ + workflowService: { + executeWait: vi.fn(), + }, +})) + +describe('readPdfPagesTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('should trigger pdfPages transcode workflow and return image outputs', async () => { + vi.mocked(prisma.asset.findUnique).mockResolvedValue({ + id: 'asset-1', + projectId: 'project-1', + } as unknown as Asset) + + vi.mocked(prisma.assetComment.findUnique).mockResolvedValue({ + id: 'comment-1', + second: 2, + annotation: [ + { + type: 'box', + color: '#ff0000', + points: [ + [0, 0], + [1, 1], + ], + }, + ], + } as unknown as AssetComment) + + vi.mocked(prisma.workflowTask.create).mockResolvedValue({ + id: 'task-1', + } as unknown as WorkflowTask) + + vi.mocked(workflowService.executeWait).mockResolvedValue({ + id: 'task-1', + status: WorkflowTaskStatus.completed, + output: { + pages: [ + { key: 'pdf_pages/page_1.webp', page: 1 }, + { key: 'pdf_pages/page_2.webp', page: 2 }, + ], + }, + } as unknown as WorkflowTask) + + vi.mocked(s3Service.getObject).mockImplementation(async (_bucket: string, key: string) => { + return { + buffer: Buffer.from(`fake-bytes-for-${key}`), + contentType: 'image/webp', + } as unknown as { buffer: Buffer; contentType: string } + }) + + const tool = createReadPdfPagesTool('asset-1', 'comment-1') + const result = await tool.execute('call-1', { start: 1, end: 2 }) + + expect(prisma.workflowTask.create).toHaveBeenCalledWith({ + data: { + assetId: 'asset-1', + projectId: 'project-1', + type: 'transcode', + status: 'pending', + payload: { + projectId: 'project-1', + pdfPages: { + start: 1, + end: 2, + commentTimestamp: 2, + annotations: [ + { + type: 'box', + color: '#ff0000', + points: [ + [0, 0], + [1, 1], + ], + }, + ], + }, + }, + }, + }) + + expect(result.content.length).toBe(2) + expect(result.content[0].type).toBe('image') + expect((result.content[0] as unknown as { data: string }).data).toBe( + Buffer.from('fake-bytes-for-pdf_pages/page_1.webp').toString('base64'), + ) + expect(result.details.sourceKeys).toEqual(['pdf_pages/page_1.webp', 'pdf_pages/page_2.webp']) + }) + + it('should return error text if asset not found', async () => { + vi.mocked(prisma.asset.findUnique).mockResolvedValue(null) + + const tool = createReadPdfPagesTool('asset-1') + const result = await tool.execute('call-1', { start: 1, end: 3 }) + + expect(result.content[0]).toEqual({ + type: 'text', + text: 'Asset with ID asset-1 not found.', + }) + }) +}) diff --git a/packages/agent/src/tools/read-pdf-pages.ts b/packages/agent/src/tools/read-pdf-pages.ts new file mode 100644 index 00000000..d164b616 --- /dev/null +++ b/packages/agent/src/tools/read-pdf-pages.ts @@ -0,0 +1,126 @@ +import { Type } from '@sinclair/typebox' +import { type AgentTool } from '@earendil-works/pi-agent-core' +import { type ImageContent } from '@earendil-works/pi-ai' +import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db' +import { s3Service } from '@shumai/core/src/s3/s3' +import { workflowService } from '@shumai/workflow-core' + +const readPdfPagesSchema = Type.Object({ + start: Type.Number({ description: 'Start page number (1-based index)' }), + end: Type.Number({ description: 'End page number (1-based index)' }), + assetId: Type.Optional( + Type.String({ + description: 'Asset ID of the PDF document (optional, defaults to current asset)', + }), + ), +}) + +export function createReadPdfPagesTool( + assetId: string, + userCommentId?: string | null, +): AgentTool { + return { + name: 'read_pdf_pages', + label: 'Read PDF Pages', + description: + 'Converts specific page range of a PDF document into 1080p WebP images for visual analysis.', + parameters: readPdfPagesSchema, + execute: async (_toolCallId, params) => { + try { + const targetAssetId = params.assetId || assetId + const asset = await prisma.asset.findUnique({ + where: { id: targetAssetId }, + }) + + if (!asset) { + return { + content: [{ type: 'text', text: `Asset with ID ${targetAssetId} not found.` }], + details: {}, + } + } + + // Fetch trigger comment details for comment page and annotations + let commentTimestamp: number | null = null + let annotations: unknown = null + if (userCommentId) { + const comment = await prisma.assetComment.findUnique({ + where: { id: userCommentId }, + }) + if (comment) { + commentTimestamp = comment.second !== null ? comment.second : null + if (comment.annotation) { + const list = comment.annotation + if (Array.isArray(list) && list.length > 0) { + annotations = list + } + } + } + } + + // Trigger transcode workflow to render PDF pages + const task = await prisma.workflowTask.create({ + data: { + assetId: targetAssetId, + projectId: asset.projectId || 'none', + type: WorkflowTaskType.transcode, + status: WorkflowTaskStatus.pending, + payload: { + projectId: asset.projectId || 'none', + pdfPages: { + start: params.start, + end: params.end, + commentTimestamp, + annotations: annotations as PrismaJson.AnnotationList, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const output = completedTask.output as any + const pages = output?.pages as Array<{ key: string; page: number }> | undefined + + if (!pages || pages.length === 0) { + return { + content: [{ type: 'text', text: 'No PDF page images were generated.' }], + details: {}, + } + } + + const content: ImageContent[] = [] + const sourceKeys: string[] = [] + const bucket = process.env.S3_BUCKET || 'shumai' + + for (const pageItem of pages) { + const { buffer, contentType } = await s3Service.getObject(bucket, pageItem.key) + content.push({ + type: 'image', + data: buffer.toString('base64'), + mimeType: contentType || 'image/webp', + }) + sourceKeys.push(pageItem.key) + } + + return { + content, + details: { + sourceKeys, + }, + } + } catch (error) { + return { + content: [ + { + type: 'text', + text: `Error reading PDF pages: ${ + error instanceof Error ? error.message : String(error) + }`, + }, + ], + details: {}, + } + } + }, + } +} diff --git a/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts b/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts index aba32e64..745706fb 100644 --- a/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts +++ b/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts @@ -49,6 +49,43 @@ describe('AgentChatPromptBuilder', () => { expect(result).not.toContain("call the 'analyze_image' tool") }) + it('should build prompt with PDF asset details and total pages', () => { + const builder = new AgentChatPromptBuilder('a1').withAssetDetails( + 'doc.pdf', + 'application/pdf', + undefined, + 15, + ) + const result = builder.build() + + expect(result).toContain('File Name: doc.pdf') + expect(result).toContain('File Type: pdf') + expect(result).toContain('Total Pages: 15') + expect(result).toContain("call the 'read_pdf_pages' tool") + expect(result).not.toContain("call the 'screenshot' tool") + expect(result).not.toContain("call the 'analyze_image' tool") + }) + + it('should build prompt with comment page for PDF asset', () => { + const builder = new AgentChatPromptBuilder('a1') + .withAssetDetails('doc.pdf', 'application/pdf', undefined, 10) + .withCommentTimestamp(3) + const result = builder.build() + + expect(result).toContain('Comment Page: 3') + }) + + it('should build prompt for PDF continuation mode with comment page', () => { + const builder = new AgentChatPromptBuilder('a1') + .withContinuation(true) + .withAssetDetails('doc.pdf', 'application/pdf', undefined, 10) + .withCommentTimestamp(4) + const result = builder.build() + + expect(result).toContain('Comment Page: 4') + expect(result).not.toContain('Comment Timestamp:') + }) + it('should build prompt with comment timestamp', () => { const builder = new AgentChatPromptBuilder('a1').withCommentTimestamp(5.5) const result = builder.build() diff --git a/packages/agent/src/workflows/agent-chat-prompt-builder.ts b/packages/agent/src/workflows/agent-chat-prompt-builder.ts index 9b4f654c..7634d771 100644 --- a/packages/agent/src/workflows/agent-chat-prompt-builder.ts +++ b/packages/agent/src/workflows/agent-chat-prompt-builder.ts @@ -1,4 +1,4 @@ -export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'other' { +export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'pdf' | 'other' { if (!mediaType) return 'other' const lower = mediaType.toLowerCase() if ( @@ -11,6 +11,9 @@ export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'oth if (lower.startsWith('video/')) { return 'video' } + if (lower === 'application/pdf' || lower.endsWith('.pdf')) { + return 'pdf' + } return 'other' } @@ -19,6 +22,7 @@ export class AgentChatPromptBuilder { private assetName?: string private mediaType?: string private videoDuration?: number + private totalPages?: number private commentTimestamp?: number private pathContext?: string private explicitMention = false @@ -55,10 +59,16 @@ export class AgentChatPromptBuilder { return this } - withAssetDetails(name: string, mediaType: string | null, duration?: number): this { + withAssetDetails( + name: string, + mediaType: string | null, + duration?: number, + totalPages?: number, + ): this { this.assetName = name this.mediaType = mediaType || undefined this.videoDuration = duration + this.totalPages = totalPages return this } @@ -78,7 +88,12 @@ export class AgentChatPromptBuilder { if (this.isContinuation) { let instruction = '' if (this.commentTimestamp !== undefined) { - instruction += `Comment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds` + const type = getSimpleMediaType(this.mediaType) + if (type === 'pdf') { + instruction += `Comment Page: ${Math.round(this.commentTimestamp)}` + } else { + instruction += `Comment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds` + } } if (this.attachedFiles.length > 0 || this.referencedAssets.length > 0) { instruction += `\n\n[Context: New Attached Files & Referenced Assets]` @@ -101,10 +116,17 @@ export class AgentChatPromptBuilder { instruction += `\nFile Type: ${type}` if (type === 'video' && this.videoDuration !== undefined) { instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds` + } else if (type === 'pdf' && this.totalPages !== undefined) { + instruction += `\nTotal Pages: ${this.totalPages}` } } if (this.commentTimestamp !== undefined) { - instruction += `\nComment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds` + const type = getSimpleMediaType(this.mediaType) + if (type === 'pdf') { + instruction += `\nComment Page: ${Math.round(this.commentTimestamp)}` + } else { + instruction += `\nComment Timestamp: ${this.commentTimestamp.toFixed(2)} seconds` + } } if (this.pathContext) { instruction += `\n\nAsset Path Context:\n${this.pathContext}` @@ -126,6 +148,8 @@ export class AgentChatPromptBuilder { instruction += `\n\nIf you need to view the image data, call the 'analyze_image' tool. It does not require any parameters.` } else if (type === 'video') { instruction += `\n\nIf you need to view visual frames or take screenshots of the video, call the 'screenshot' tool. You must specify the 'start' (seconds), 'end' (seconds), and 'count' (number of screenshots) parameters.` + } else if (type === 'pdf') { + instruction += `\n\nIf you need to view pages of the PDF document, call the 'read_pdf_pages' tool. You must specify the 'start' (page number) and 'end' (page number) parameters.` } } diff --git a/packages/agent/src/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts index fb8e65c9..7e201a17 100644 --- a/packages/agent/src/workflows/agent-chat.ts +++ b/packages/agent/src/workflows/agent-chat.ts @@ -124,6 +124,7 @@ export async function agentChat(task: WorkflowTask): Promise { // eslint-disable-next-line @typescript-eslint/no-explicit-any const mediaInfo = asset.media as any const duration = mediaInfo?.duration + const totalPages = mediaInfo?.frames || mediaInfo?.metadata?.totalFrames // Resolve Attached Files Details const attachedFileDetailsList: string[] = [] @@ -163,7 +164,7 @@ export async function agentChat(task: WorkflowTask): Promise { const instruction = new AgentChatPromptBuilder(asset.id) .withContinuation(!isNewChat) .withPathContext(pathContext) - .withAssetDetails(asset.name, asset.mediaType, duration) + .withAssetDetails(asset.name, asset.mediaType, duration, totalPages) .withCommentTimestamp(commentTimestamp) .withExplicitMention(payload.agent?.explicitMention) .withAttachedFiles(attachedFileDetailsList) diff --git a/packages/core/src/transcode/transcode.test.ts b/packages/core/src/transcode/transcode.test.ts index e8305fc4..cc3456b5 100644 --- a/packages/core/src/transcode/transcode.test.ts +++ b/packages/core/src/transcode/transcode.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { transcodeService } from './transcode' +import { s3Service } from '@shumai/core/src/s3/s3' import * as path from 'path' import * as child_process from 'child_process' import * as fs from 'fs' @@ -7,6 +8,13 @@ import { WorkflowTask } from '@shumai/db' import { setupTestDbHooks } from '@shumai/db/test' import sharp from 'sharp' +vi.mock('@shumai/core/src/s3/s3', () => ({ + s3Service: { + downloadToFile: vi.fn(), + putObject: vi.fn(), + }, +})) + // Mock child_process vi.mock('child_process', () => ({ execFile: vi.fn(), @@ -17,7 +25,10 @@ vi.mock('sharp', () => { const mockSharp = { resize: vi.fn().mockReturnThis(), webp: vi.fn().mockReturnThis(), - toFile: vi.fn().mockResolvedValue({}), + toFile: vi.fn().mockImplementation(async (filePath: string) => { + fs.writeFileSync(filePath, 'fake-webp-data') + return {} + }), metadata: vi.fn().mockResolvedValue({ width: 800, height: 600, format: 'png' }), } const sharpFunc = vi.fn(() => mockSharp) @@ -428,4 +439,41 @@ describe('TranscodeService', () => { expect(info.originalHeight).toBe(600) expect(info.hasAudio).toBe(false) }) + + it('should render PDF pages and upload to S3', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(child_process.execFile as any).mockImplementation( + ( + cmd: string, + args: string[], + cb: (err: Error | null, res: { stdout: string; stderr: string }) => void, + ) => { + if (cmd === 'pdftoppm') { + const pagePrefix = args[args.length - 1] + const pagePath = `${pagePrefix}-1.png` + fs.writeFileSync(pagePath, 'fake-png-data') + cb(null, { stdout: '', stderr: '' }) + } else { + cb(null, { stdout: '', stderr: '' }) + } + }, + ) + + vi.mocked(s3Service.downloadToFile).mockResolvedValue(undefined) + vi.mocked(s3Service.putObject).mockResolvedValue( + {} as unknown as Awaited>, + ) + + const result = await transcodeService.renderPdfPages({ + assetKey: 'projects/p1/doc.pdf', + assetId: 'asset-1', + start: 1, + end: 1, + }) + + expect(result.length).toBe(1) + expect(result[0].page).toBe(1) + expect(result[0].key).toContain('files/asset-1/pdf_pages/doc-page-1-') + expect(s3Service.putObject).toHaveBeenCalled() + }) }) diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts index 0d8823eb..9bd11fc9 100644 --- a/packages/core/src/transcode/transcode.ts +++ b/packages/core/src/transcode/transcode.ts @@ -1,4 +1,5 @@ import { s3Service } from '@shumai/core/src/s3/s3' +import { stemFromKey } from '@shumai/core/src/utils/filename' import { prisma, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' import '@shumai/db/src/prisma-json-types' import { execFile } from 'child_process' @@ -664,6 +665,111 @@ export class TranscodeService { } } + async renderPdfPages(params: { + assetKey: string + assetId: string + start: number + end: number + commentTimestamp?: number | null + annotations?: PrismaJson.AnnotationList | null + }): Promise> { + const bucket = process.env.S3_BUCKET || 'shumai' + const tmpDir = this.createTempDir('pdf-pages-') + const pdfPath = path.join(tmpDir, path.basename(params.assetKey)) + + try { + // 1. Download PDF file + await s3Service.downloadToFile(bucket, params.assetKey, pdfPath) + + // 2. Render pages via pdftoppm (-png -f start -l end) + const pagePrefix = path.join(tmpDir, 'page') + try { + await execFileAsync('pdftoppm', [ + '-png', + '-f', + params.start.toString(), + '-l', + params.end.toString(), + pdfPath, + pagePrefix, + ]) + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + if (msg.includes('pdftoppm') || msg.includes('ENOENT') || msg.includes('not found')) { + throw new Error( + `pdftoppm executable not found in $PATH. Please install poppler-utils / poppler. (${msg})`, + { cause: err }, + ) + } + throw err + } + + const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith('.png')) + if (files.length === 0) { + throw new Error('pdftoppm produced no image outputs') + } + + files.sort((a, b) => { + const numA = parseInt(a.replace(/[^0-9]/g, ''), 10) + const numB = parseInt(b.replace(/[^0-9]/g, ''), 10) + return numA - numB + }) + + const results: Array<{ key: string; page: number }> = [] + const stem = stemFromKey(params.assetKey) + + for (let i = 0; i < files.length; i++) { + const pageNum = params.start + i + const localPngPath = path.join(tmpDir, files[i]) + const webpName = `${stem}-page-${pageNum}-${ulid()}.webp` + const localWebpPath = path.join(tmpDir, webpName) + + // Convert to 1080p WebP (fit inside 1920x1080) + await sharp(localPngPath, { limitInputPixels: false }) + .resize(1920, 1080, { fit: 'inside', withoutEnlargement: false }) + .webp({ quality: 85 }) + .toFile(localWebpPath) + + // Overlay annotations if comment page matches pageNum + const commentPage = + params.commentTimestamp !== undefined && params.commentTimestamp !== null + ? Math.round(params.commentTimestamp) + : null + + if ( + commentPage !== null && + pageNum === commentPage && + params.annotations && + params.annotations.length > 0 + ) { + const meta = await sharp(localWebpPath, { limitInputPixels: false }).metadata() + const width = meta.width || 1920 + const height = meta.height || 1080 + + const svgStr = renderAnnotationsToSvg(width, height, params.annotations) + const tempCompositedPath = path.join(tmpDir, `composite-${webpName}`) + + await sharp(localWebpPath, { limitInputPixels: false }) + .composite([{ input: Buffer.from(svgStr), top: 0, left: 0 }]) + .toFile(tempCompositedPath) + + fs.renameSync(tempCompositedPath, localWebpPath) + } + + // Upload to S3 + const s3Key = `files/${params.assetId}/pdf_pages/${webpName}` + const webpBuffer = fs.readFileSync(localWebpPath) + await s3Service.putObject(bucket, s3Key, webpBuffer, webpBuffer.length, 'image/webp') + + results.push({ key: s3Key, page: pageNum }) + } + + return results + } finally { + this.removeDir(tmpDir) + } + } + async overlayAnnotations(params: { assetKey: string assetId: string diff --git a/packages/db/src/prisma-json-types.ts b/packages/db/src/prisma-json-types.ts index 5bf3b47b..70bcabe3 100644 --- a/packages/db/src/prisma-json-types.ts +++ b/packages/db/src/prisma-json-types.ts @@ -255,6 +255,13 @@ declare global { annotations?: AnnotationList | null } + export interface PdfPagesSpec { + start: number + end: number + commentTimestamp?: number | null + annotations?: AnnotationList | null + } + export interface WorkflowTaskPayload { projectId: string transcode?: TaskSpec @@ -265,6 +272,7 @@ declare global { agentToolCall?: AgentToolCallPayload screenshot?: ScreenshotSpec imageAnnotation?: ImageAnnotationSpec + pdfPages?: PdfPagesSpec } // ---------------------------------------------------------------------- diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index a95a9368..1f011a43 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -12,6 +12,7 @@ vi.mock('child_process', () => ({ import { getMediaInfoActivity, overlayAnnotationsActivity, + renderPdfPagesActivity, takeScreenshotsActivity, transcodeVideoActivity, transcodeAudioActivity, @@ -62,6 +63,7 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({ removeDir: vi.fn(), takeScreenshots: vi.fn(), overlayAnnotations: vi.fn(), + renderPdfPages: vi.fn(), }, })) @@ -321,6 +323,31 @@ describe('Transcode Activities', () => { expect(result).toBe('annotations/ann.webp') }) + it('should call transcodeService.renderPdfPages', async () => { + vi.mocked(transcodeService.renderPdfPages).mockResolvedValue([ + { key: 'pdf_pages/doc-page-1.webp', page: 1 }, + ]) + + const result = await renderPdfPagesActivity({ + assetKey: 'doc.pdf', + assetId: 'asset-pdf', + start: 1, + end: 1, + commentTimestamp: 1, + annotations: [], + }) + + expect(transcodeService.renderPdfPages).toHaveBeenCalledWith({ + assetKey: 'doc.pdf', + assetId: 'asset-pdf', + start: 1, + end: 1, + commentTimestamp: 1, + annotations: [], + }) + expect(result).toEqual([{ key: 'pdf_pages/doc-page-1.webp', page: 1 }]) + }) + describe('Non-retryable Error Handling', () => { it('should throw non-retryable ApplicationFailure when downloadMediaToTmpActivity fails with ENOENT', async () => { vi.mocked(s3Service.downloadToFile).mockRejectedValue({ diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index c3c1aa6d..24f7e1f4 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -606,6 +606,36 @@ export async function overlayAnnotationsActivity(params: { } } +export async function renderPdfPagesActivity(params: { + assetKey: string + assetId: string + start: number + end: number + commentTimestamp?: number | null + annotations?: PrismaJson.AnnotationList | null +}): Promise> { + try { + return await transcodeService.renderPdfPages(params) + } catch (err) { + const { code, message } = getErrorDetails(err) + const lowerMsg = message.toLowerCase() + if ( + code === 'ENOENT' || + lowerMsg.includes('enoent') || + lowerMsg.includes('nosuchkey') || + lowerMsg.includes('pdftoppm') || + lowerMsg.includes('sharp') + ) { + throw ApplicationFailure.create({ + message: `PDF page rendering failed: ${message}`, + nonRetryable: true, + cause: err instanceof Error ? err : undefined, + }) + } + throw err + } +} + export interface CreateEmbeddingTaskIfEnabledParams { assetId: string teamId: string | null diff --git a/packages/transcode/src/workflows/transcode.test.ts b/packages/transcode/src/workflows/transcode.test.ts index 2e031c94..cba32b0e 100644 --- a/packages/transcode/src/workflows/transcode.test.ts +++ b/packages/transcode/src/workflows/transcode.test.ts @@ -42,6 +42,9 @@ describe('Transcode Workflow', () => { overlayAnnotationsActivity: Object.assign(vi.fn(), { _activityName: 'overlayAnnotationsActivity', }), + renderPdfPagesActivity: Object.assign(vi.fn(), { + _activityName: 'renderPdfPagesActivity', + }), createEmbeddingTaskIfEnabledActivity: Object.assign(vi.fn(), { _activityName: 'createEmbeddingTaskIfEnabledActivity', }), @@ -478,6 +481,68 @@ describe('Transcode Workflow', () => { }) }) + it('should run pdfPages workflow successfully', async () => { + const task: WorkflowTask = { + id: 'task-pdf-pages', + assetId: 'asset-pdf', + type: WorkflowTaskType.transcode, + status: WorkflowTaskStatus.pending, + payload: { + projectId: 'proj-1', + pdfPages: { + start: 1, + end: 2, + commentTimestamp: 2, + annotations: [], + }, + }, + output: null, + sessionId: null, + createdAt: new Date(), + updatedAt: new Date(), + heartbeat: null, + teamId: 'team-1', + projectId: 'proj-1', + uid: 'task-uid', + model: null, + inputTokens: 0, + outputTokens: 0, + } + + mockActivities.getAssetActivity.mockResolvedValue({ + id: 'asset-pdf', + storageKey: { key: 'doc.pdf' }, + mediaType: 'application/pdf', + }) + + mockActivities.renderPdfPagesActivity.mockResolvedValue([ + { key: 'pdf_pages/doc-page-1.webp', page: 1 }, + { key: 'pdf_pages/doc-page-2.webp', page: 2 }, + ]) + + await transcodeMedia(task) + + expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalledWith({ + assetKey: 'doc.pdf', + assetId: 'asset-pdf', + start: 1, + end: 2, + commentTimestamp: 2, + annotations: [], + }) + + expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({ + taskId: 'task-pdf-pages', + status: WorkflowTaskStatus.completed, + output: { + pages: [ + { key: 'pdf_pages/doc-page-1.webp', page: 1 }, + { key: 'pdf_pages/doc-page-2.webp', page: 2 }, + ], + }, + }) + }) + it('should run audio transcoding workflow successfully', async () => { const task: WorkflowTask = { id: 'task-audio', diff --git a/packages/transcode/src/workflows/transcode.ts b/packages/transcode/src/workflows/transcode.ts index 7be0ae6f..da94c0f4 100644 --- a/packages/transcode/src/workflows/transcode.ts +++ b/packages/transcode/src/workflows/transcode.ts @@ -19,6 +19,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise { cleanupTmpDirActivity, takeScreenshotsActivity, overlayAnnotationsActivity, + renderPdfPagesActivity, createEmbeddingTaskIfEnabledActivity, } = getActivities() @@ -37,6 +38,34 @@ export async function transcodeMedia(task: WorkflowTask): Promise { const payload = task.payload + if (payload?.pdfPages) { + const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId) + const key = asset?.storageKey?.key + if (!asset || !key) { + throw ApplicationFailure.create({ + message: 'Asset not found or has no key', + nonRetryable: true, + }) + } + + const pages = await executeActivity(workerQueue, renderPdfPagesActivity, { + assetKey: key, + assetId: asset.id, + start: payload.pdfPages.start, + end: payload.pdfPages.end, + commentTimestamp: payload.pdfPages.commentTimestamp, + annotations: payload.pdfPages.annotations, + }) + + // Update Task Status to completed with output + await executeActivity(workerQueue, updateTaskStatusActivity, { + taskId: task.id, + status: 'completed', + output: { pages }, + }) + return + } + if (payload?.screenshot) { const asset = await executeActivity(workerQueue, getAssetActivity, task.assetId) const key = asset?.storageKey?.key diff --git a/packages/webui/components/settings/AgentFormDialog.tsx b/packages/webui/components/settings/AgentFormDialog.tsx index 7049a450..aee634c1 100644 --- a/packages/webui/components/settings/AgentFormDialog.tsx +++ b/packages/webui/components/settings/AgentFormDialog.tsx @@ -76,6 +76,11 @@ const BUILTIN_TOOLS = [ name: () => m.agent_tool_screenshot_name(), description: () => m.agent_tool_screenshot_desc(), }, + { + id: 'read_pdf_pages', + name: () => m.agent_tool_read_pdf_pages_name(), + description: () => m.agent_tool_read_pdf_pages_desc(), + }, { id: 'list_assets', name: () => m.agent_tool_list_assets_name(), diff --git a/packages/webui/messages/en.json b/packages/webui/messages/en.json index d863150e..ea595663 100644 --- a/packages/webui/messages/en.json +++ b/packages/webui/messages/en.json @@ -389,6 +389,8 @@ "agent_tool_analyze_image_desc": "Allows the agent to view and analyze the visual contents of an image file.", "agent_tool_screenshot_name": "Video Frame Extraction", "agent_tool_screenshot_desc": "Allows the agent to extract still image frames from video files for analysis.", + "agent_tool_read_pdf_pages_name": "Read PDF Pages", + "agent_tool_read_pdf_pages_desc": "Allows the agent to convert and view specific pages of a PDF document as images.", "agent_tool_list_assets_name": "Browse Shumai", "agent_tool_list_assets_desc": "Allows the agent to list folders and browse files in Shumai.", "agent_tool_create_folder_name": "Create Folder", diff --git a/packages/webui/messages/zh.json b/packages/webui/messages/zh.json index c9385898..7cab61d7 100644 --- a/packages/webui/messages/zh.json +++ b/packages/webui/messages/zh.json @@ -389,6 +389,8 @@ "agent_tool_analyze_image_desc": "允许智能体查看和分析图片文件的视觉内容。", "agent_tool_screenshot_name": "提取视频帧", "agent_tool_screenshot_desc": "允许智能体从视频文件中提取静态画面进行分析。", + "agent_tool_read_pdf_pages_name": "读取 PDF 页面", + "agent_tool_read_pdf_pages_desc": "允许智能体将 PDF 文件的指定页面转换为图片并查看。", "agent_tool_list_assets_name": "浏览 Shumai", "agent_tool_list_assets_desc": "允许智能体列出文件夹并浏览 Shumai 中的文件。", "agent_tool_create_folder_name": "创建文件夹", diff --git a/packages/webui/paraglide/messages/_index.d.ts b/packages/webui/paraglide/messages/_index.d.ts index d7805ac4..4b4955c6 100644 --- a/packages/webui/paraglide/messages/_index.d.ts +++ b/packages/webui/paraglide/messages/_index.d.ts @@ -371,6 +371,8 @@ export * from "./agent_tool_analyze_image_name.js"; export * from "./agent_tool_analyze_image_desc.js"; export * from "./agent_tool_screenshot_name.js"; export * from "./agent_tool_screenshot_desc.js"; +export * from "./agent_tool_read_pdf_pages_name.js"; +export * from "./agent_tool_read_pdf_pages_desc.js"; export * from "./agent_tool_list_assets_name.js"; export * from "./agent_tool_list_assets_desc.js"; export * from "./agent_tool_create_folder_name.js"; diff --git a/packages/webui/paraglide/messages/_index.js b/packages/webui/paraglide/messages/_index.js index 504d6ccc..dae99dda 100644 --- a/packages/webui/paraglide/messages/_index.js +++ b/packages/webui/paraglide/messages/_index.js @@ -373,6 +373,8 @@ export * from './agent_tool_analyze_image_name.js' export * from './agent_tool_analyze_image_desc.js' export * from './agent_tool_screenshot_name.js' export * from './agent_tool_screenshot_desc.js' +export * from './agent_tool_read_pdf_pages_name.js' +export * from './agent_tool_read_pdf_pages_desc.js' export * from './agent_tool_list_assets_name.js' export * from './agent_tool_list_assets_desc.js' export * from './agent_tool_create_folder_name.js' diff --git a/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.d.ts b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.d.ts new file mode 100644 index 00000000..3393e629 --- /dev/null +++ b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.d.ts @@ -0,0 +1,16 @@ +/** +* | output | +* | --- | +* | "Allows the agent to convert and view specific pages of a PDF document as images." | +* +* @param {Agent_Tool_Read_Pdf_Pages_DescInputs} inputs +* @param {{ locale?: "en" | "zh" }} options +* @returns {LocalizedString} +*/ +export const agent_tool_read_pdf_pages_desc: ((inputs?: Agent_Tool_Read_Pdf_Pages_DescInputs, options?: { + locale?: "en" | "zh"; +}) => LocalizedString) & import("../runtime.js").MessageMetadata; +export type LocalizedString = import("../runtime.js").LocalizedString; +export type Agent_Tool_Read_Pdf_Pages_DescInputs = {}; diff --git a/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.js b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.js new file mode 100644 index 00000000..78a91df3 --- /dev/null +++ b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_desc.js @@ -0,0 +1,29 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Agent_Tool_Read_Pdf_Pages_DescInputs */ + +const en_agent_tool_read_pdf_pages_desc = /** @type {(inputs: Agent_Tool_Read_Pdf_Pages_DescInputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Allows the agent to convert and view specific pages of a PDF document as images.`) +}; + +const zh_agent_tool_read_pdf_pages_desc = /** @type {(inputs: Agent_Tool_Read_Pdf_Pages_DescInputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`允许智能体将 PDF 文件的指定页面转换为图片并查看。`) +}; + +/** +* | output | +* | --- | +* | "Allows the agent to convert and view specific pages of a PDF document as images." | +* +* @param {Agent_Tool_Read_Pdf_Pages_DescInputs} inputs +* @param {{ locale?: "en" | "zh" }} options +* @returns {LocalizedString} +*/ +export const agent_tool_read_pdf_pages_desc = /** @type {((inputs?: Agent_Tool_Read_Pdf_Pages_DescInputs, options?: { locale?: "en" | "zh" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "en") return en_agent_tool_read_pdf_pages_desc(inputs) + return zh_agent_tool_read_pdf_pages_desc(inputs) +}); \ No newline at end of file diff --git a/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.d.ts b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.d.ts new file mode 100644 index 00000000..ca1bacbc --- /dev/null +++ b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.d.ts @@ -0,0 +1,16 @@ +/** +* | output | +* | --- | +* | "Read PDF Pages" | +* +* @param {Agent_Tool_Read_Pdf_Pages_NameInputs} inputs +* @param {{ locale?: "en" | "zh" }} options +* @returns {LocalizedString} +*/ +export const agent_tool_read_pdf_pages_name: ((inputs?: Agent_Tool_Read_Pdf_Pages_NameInputs, options?: { + locale?: "en" | "zh"; +}) => LocalizedString) & import("../runtime.js").MessageMetadata; +export type LocalizedString = import("../runtime.js").LocalizedString; +export type Agent_Tool_Read_Pdf_Pages_NameInputs = {}; diff --git a/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.js b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.js new file mode 100644 index 00000000..afffea8f --- /dev/null +++ b/packages/webui/paraglide/messages/agent_tool_read_pdf_pages_name.js @@ -0,0 +1,29 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Agent_Tool_Read_Pdf_Pages_NameInputs */ + +const en_agent_tool_read_pdf_pages_name = /** @type {(inputs: Agent_Tool_Read_Pdf_Pages_NameInputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Read PDF Pages`) +}; + +const zh_agent_tool_read_pdf_pages_name = /** @type {(inputs: Agent_Tool_Read_Pdf_Pages_NameInputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`读取 PDF 页面`) +}; + +/** +* | output | +* | --- | +* | "Read PDF Pages" | +* +* @param {Agent_Tool_Read_Pdf_Pages_NameInputs} inputs +* @param {{ locale?: "en" | "zh" }} options +* @returns {LocalizedString} +*/ +export const agent_tool_read_pdf_pages_name = /** @type {((inputs?: Agent_Tool_Read_Pdf_Pages_NameInputs, options?: { locale?: "en" | "zh" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "en") return en_agent_tool_read_pdf_pages_name(inputs) + return zh_agent_tool_read_pdf_pages_name(inputs) +}); \ No newline at end of file