From d052c56d3923c98f0b8c3c58fd623afe8ef04db3 Mon Sep 17 00:00:00 2001 From: yiling Date: Sat, 25 Jul 2026 10:46:20 +0800 Subject: [PATCH 1/3] fix(transcode): handle timestamp rounding tolerance for screenshot annotations --- packages/agent/src/tools/screenshot.ts | 13 + packages/core/src/transcode/transcode.test.ts | 91 +++++++ packages/core/src/transcode/transcode.ts | 64 ++++- .../workflow/take-video-screenshots.test.ts | 227 ++++++++++++++++++ 4 files changed, 387 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/tools/screenshot.ts b/packages/agent/src/tools/screenshot.ts index f2030a1f..6e9896e8 100644 --- a/packages/agent/src/tools/screenshot.ts +++ b/packages/agent/src/tools/screenshot.ts @@ -2,6 +2,7 @@ import { Type } from '@sinclair/typebox' import { type AgentTool } from '@earendil-works/pi-agent-core' import { type ImageContent } from '@earendil-works/pi-ai' import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db' +import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' import { workflowService } from '@shumai/workflow-core' import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz' @@ -65,6 +66,18 @@ export function createScreenshotTool( } } + logger.info( + { + assetId, + userId, + userCommentId, + commentTimestamp, + hasAnnotations: Boolean(annotations), + params, + }, + '[screenshotTool] Executing screenshot tool call', + ) + // Trigger transcode workflow to take screenshots const task = await prisma.workflowTask.create({ data: { diff --git a/packages/core/src/transcode/transcode.test.ts b/packages/core/src/transcode/transcode.test.ts index 6d1a463d..318a5aa3 100644 --- a/packages/core/src/transcode/transcode.test.ts +++ b/packages/core/src/transcode/transcode.test.ts @@ -646,6 +646,97 @@ describe('TranscodeService', () => { expect(b).toBeLessThan(100) }) + it('should snap commentTimestamp and overlay annotations when start/end has rounding mismatch', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(child_process.execFile as any).mockImplementation( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (file: string, args: string[], cb: any) => { + if (file === 'ffmpeg') { + const outPath = args[args.length - 1] + fs.writeFileSync(outPath, 'fake-webp-image') + } + cb(null, { stdout: '', stderr: '' }) + }, + ) + + const overlaySpy = vi + .spyOn(transcodeService, 'overlayAnnotationsOnBuffer') + .mockResolvedValue(Buffer.from('composited-buffer')) + + const annotations: PrismaJson.AnnotationList = [ + { + type: 'box', + color: '#ff0000', + points: [ + [0.2, 0.2], + [0.8, 0.8], + ], + }, + ] + + const results = await transcodeService.takeScreenshots({ + assetKey: 'test/video.mp4', + assetId: 'asset-123', + start: 4.57, + end: 4.57, + count: 1, + commentTimestamp: 4.566666666666667, + annotations, + }) + + expect(results).toHaveLength(1) + expect(results[0].timestamp).toBe(4.566666666666667) + expect(overlaySpy).toHaveBeenCalledWith(expect.any(Buffer), annotations) + + overlaySpy.mockRestore() + }) + + it('should overlay annotation on ONLY the single snapped frame in a multi-screenshot range', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(child_process.execFile as any).mockImplementation( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (file: string, args: string[], cb: any) => { + if (file === 'ffmpeg') { + const outPath = args[args.length - 1] + fs.writeFileSync(outPath, 'fake-webp-image') + } + cb(null, { stdout: '', stderr: '' }) + }, + ) + + const overlaySpy = vi + .spyOn(transcodeService, 'overlayAnnotationsOnBuffer') + .mockResolvedValue(Buffer.from('composited-buffer')) + + const annotations: PrismaJson.AnnotationList = [ + { + type: 'box', + color: '#ff0000', + points: [ + [0.2, 0.2], + [0.8, 0.8], + ], + }, + ] + + const results = await transcodeService.takeScreenshots({ + assetKey: 'test/video.mp4', + assetId: 'asset-123', + start: 0, + end: 1, + count: 30, + commentTimestamp: 0.566666666666667, + annotations, + }) + + expect(results).toHaveLength(30) + // Exactly 1 overlay call for the snapped timestamp out of 30 frames + expect(overlaySpy).toHaveBeenCalledTimes(1) + expect(overlaySpy).toHaveBeenCalledWith(expect.any(Buffer), annotations) + + overlaySpy.mockRestore() + }) + it('should generate PDF from text file including CJK characters', async () => { const txtFile = path.join(tempDir, 'test.txt') const pdfFile = path.join(tempDir, 'output.pdf') diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts index 45f8542e..9fa5dd6a 100644 --- a/packages/core/src/transcode/transcode.ts +++ b/packages/core/src/transcode/transcode.ts @@ -1,3 +1,4 @@ +import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' import { stemFromKey } from '@shumai/core/src/utils/filename' import { prisma, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' @@ -911,6 +912,20 @@ export class TranscodeService { const videoPath = path.join(tmpDir, path.basename(params.assetKey)) try { + logger.info( + { + assetId: params.assetId, + assetKey: params.assetKey, + start: params.start, + end: params.end, + count: params.count, + commentTimestamp: params.commentTimestamp, + hasAnnotations: Boolean(params.annotations && params.annotations.length > 0), + annotationsCount: params.annotations?.length ?? 0, + }, + '[takeScreenshots] Starting screenshot extraction', + ) + // 1. Download video await s3Service.downloadToFile(bucket, params.assetKey, videoPath) @@ -923,10 +938,18 @@ export class TranscodeService { timestamps = Array.from({ length: params.count }, (_, i) => params.start + i * step) } - // 3. Snap closest timestamp to commentTimestamp if within range + logger.info( + { initialTimestamps: timestamps, count: params.count }, + '[takeScreenshots] Generated initial sampling timestamps', + ) + + // 3. Snap closest timestamp to commentTimestamp if within range (with 100ms tolerance) const commentTimestamp = params.commentTimestamp + const EPSILON = 0.1 if (commentTimestamp !== undefined && commentTimestamp !== null) { - if (commentTimestamp >= params.start && commentTimestamp <= params.end) { + const inRange = + commentTimestamp >= params.start - EPSILON && commentTimestamp <= params.end + EPSILON + if (inRange) { let closestIdx = 0 let minDiff = Math.abs(timestamps[0] - commentTimestamp) for (let i = 1; i < timestamps.length; i++) { @@ -936,8 +959,30 @@ export class TranscodeService { closestIdx = i } } + const oldTs = timestamps[closestIdx] timestamps[closestIdx] = commentTimestamp + logger.info( + { + commentTimestamp, + snappedIdx: closestIdx, + replacedTimestamp: oldTs, + newTimestamp: commentTimestamp, + resultingTimestamps: timestamps, + }, + '[takeScreenshots] Snapped commentTimestamp to closest timestamp index', + ) + } else { + logger.warn( + { + commentTimestamp, + start: params.start, + end: params.end, + }, + '[takeScreenshots] commentTimestamp is OUT OF RANGE [start, end], skipping snap', + ) } + } else { + logger.info('[takeScreenshots] No commentTimestamp provided') } const results: Array<{ key: string; timestamp: number }> = [] @@ -964,14 +1009,17 @@ export class TranscodeService { ] await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args]) - // 5. Overlay annotations if timestamp matches commentTimestamp exactly - if ( + const isMatch = commentTimestamp !== undefined && commentTimestamp !== null && - t === commentTimestamp && - params.annotations && - params.annotations.length > 0 - ) { + Math.abs(t - commentTimestamp) < 1e-4 + + // 5. Overlay annotations if timestamp matches commentTimestamp (within float tolerance) + if (isMatch && params.annotations && params.annotations.length > 0) { + logger.info( + { timestamp: t, commentTimestamp, annotationsCount: params.annotations.length }, + '[takeScreenshots] Overlaying annotations onto frame', + ) const shotBuffer = fs.readFileSync(localShotPath) const composited = await this.overlayAnnotationsOnBuffer(shotBuffer, params.annotations) fs.writeFileSync(localShotPath, composited) diff --git a/packages/e2e/workflow/take-video-screenshots.test.ts b/packages/e2e/workflow/take-video-screenshots.test.ts index c60f758a..54516ad8 100644 --- a/packages/e2e/workflow/take-video-screenshots.test.ts +++ b/packages/e2e/workflow/take-video-screenshots.test.ts @@ -7,6 +7,7 @@ import { s3Service } from '@shumai/core/src/s3/s3' import { fileURLToPath } from 'url' import * as path from 'path' import * as fs from 'fs' +import sharp from 'sharp' const currentDir = path.dirname(fileURLToPath(import.meta.url)) const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts') @@ -126,5 +127,231 @@ describe.each(['local', 'temporal'] as const)( expect(output.screenshots.length).toBe(1) expect(output.screenshots[0].key).toContain('screenshots/') }, 50000) + + it('should draw annotation box correctly for 5 random comment timestamps', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Video Screenshot Annotation Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Video Screenshot Annotation Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/video-shot-ann.mp4', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'video-shot-ann.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const mp4Path = path.join(fixturesDir, 'small.mp4') + const mp4Buffer = fs.readFileSync(mp4Path) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/video-shot-ann.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + + // 3. Generate 5 random comment timestamps between 0.05s and 0.95s + const randomTimestamps: number[] = [] + for (let i = 0; i < 5; i++) { + const randomTs = 0.05 + Math.random() * 0.9 + randomTimestamps.push(randomTs) + } + + console.log('Testing random timestamps:', randomTimestamps) + + const boxAnnotation = [ + { + type: 'box' as const, + color: '#ff0000', + points: [ + [0.2, 0.2], + [0.8, 0.8], + ], + }, + ] + + for (const commentTimestamp of randomTimestamps) { + // Create Workflow Task with commentTimestamp & annotations + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_screenshot', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + screenshot: { + start: 0, + end: 1, + count: 5, + commentTimestamp, + annotations: boxAnnotation, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const output = completedTask.output as unknown as { + screenshots: { key: string; timestamp: number }[] + } + expect(output).toBeDefined() + expect(output.screenshots).toBeDefined() + + // Find the screenshot matching commentTimestamp + const matchingShot = output.screenshots.find( + (s) => Math.abs(s.timestamp - commentTimestamp) < 1e-6, + ) + expect(matchingShot).toBeDefined() + + // Download screenshot buffer from S3 + const s3Obj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + matchingShot!.key, + ) + const shotBuffer = s3Obj.buffer + + // Verify pixel data using sharp to check if red box annotation was overlaid + const { data, info } = await sharp(shotBuffer).raw().toBuffer({ resolveWithObject: true }) + + // Target pixel inside box (e.g. x = 50%, y = 20% near top edge of red box) + const pixelX = Math.floor(info.width * 0.5) + const pixelY = Math.floor(info.height * 0.2) + const offset = (pixelY * info.width + pixelX) * info.channels + + const r = data[offset] + const g = data[offset + 1] + const b = data[offset + 2] + + // Red box color #ff0000 should result in high red channel and low green/blue channels + expect(r).toBeGreaterThan(150) + expect(g).toBeLessThan(120) + expect(b).toBeLessThan(120) + } + }, 90000) + + it('should overlay annotation when commentTimestamp has rounding mismatch with start/end', async () => { + // 1. Seed Database + const team = await prisma.team.create({ + data: { name: 'E2E Video Screenshot Rounding Team' }, + }) + + const project = await prisma.project.create({ + data: { name: 'E2E Video Screenshot Rounding Project', teamId: team.id }, + }) + + const storageKey = await prisma.storageKey.create({ + data: { + key: 'projects/e2e/video-shot-rounding.mp4', + }, + }) + + const asset = await prisma.asset.create({ + data: { + name: 'video-shot-rounding.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + // 2. Seed S3 Storage from Fixture + const mp4Path = path.join(fixturesDir, 'small.mp4') + const mp4Buffer = fs.readFileSync(mp4Path) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/video-shot-rounding.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + + // Reproduce log scenario: exact timestamp is 0.566666666666667, Agent requested rounded 0.57 + const commentTimestamp = 0.566666666666667 + const start = 0.57 + const end = 0.57 + + const boxAnnotation = [ + { + type: 'box' as const, + color: '#ff0000', + points: [ + [0.2, 0.2], + [0.8, 0.8], + ], + }, + ] + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_screenshot', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + screenshot: { + start, + end, + count: 1, + commentTimestamp, + annotations: boxAnnotation, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const output = completedTask.output as unknown as { + screenshots: { key: string; timestamp: number }[] + } + expect(output).toBeDefined() + expect(output.screenshots).toBeDefined() + expect(output.screenshots.length).toBe(1) + + const s3Obj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + output.screenshots[0].key, + ) + const shotBuffer = s3Obj.buffer + + const { data, info } = await sharp(shotBuffer).raw().toBuffer({ resolveWithObject: true }) + + const pixelX = Math.floor(info.width * 0.5) + const pixelY = Math.floor(info.height * 0.2) + const offset = (pixelY * info.width + pixelX) * info.channels + + const r = data[offset] + const g = data[offset + 1] + const b = data[offset + 2] + + expect(r).toBeGreaterThan(150) + expect(g).toBeLessThan(120) + expect(b).toBeLessThan(120) + }, 50000) }, ) From 6226090fd1fac742e5b4c512eab7395aadcf8565 Mon Sep 17 00:00:00 2001 From: yiling Date: Sat, 25 Jul 2026 11:17:32 +0800 Subject: [PATCH 2/3] fix(e2e): add sharp dependency to packages/e2e --- bun.lock | 91 +++++++++++++++++++++++++++++++++------ packages/e2e/package.json | 15 ++++--- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/bun.lock b/bun.lock index af998c22..e199c661 100644 --- a/bun.lock +++ b/bun.lock @@ -33,7 +33,7 @@ }, "apps/agent": { "name": "@shumai/agent-app", - "version": "0.2.0", + "version": "0.2.2", "bin": { "shumai-agent": "./src/index.ts", }, @@ -45,7 +45,7 @@ }, "apps/cli": { "name": "@shumai/cli", - "version": "0.2.0", + "version": "0.2.2", "bin": { "shumai-cli": "./src/index.ts", }, @@ -60,7 +60,7 @@ }, "apps/transcode": { "name": "@shumai/transcode-app", - "version": "0.2.0", + "version": "0.2.2", "bin": { "shumai-transcode": "./src/index.ts", }, @@ -72,7 +72,7 @@ }, "apps/web": { "name": "@shumai/web-app", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@shumai/agent": "workspace:*", "@shumai/api": "workspace:*", @@ -92,7 +92,7 @@ }, "packages/agent": { "name": "@shumai/agent", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.52", "@earendil-works/pi-agent-core": "0.78.0", @@ -117,7 +117,7 @@ }, "packages/api": { "name": "@shumai/api", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@hono/zod-validator": "0.7.6", "@shumai/core": "workspace:*", @@ -131,7 +131,7 @@ }, "packages/core": { "name": "@shumai/core", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@earendil-works/pi-agent-core": "0.78.0", "@shumai/db": "workspace:*", @@ -162,7 +162,7 @@ }, "packages/db": { "name": "@shumai/db", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@earendil-works/pi-agent-core": "0.78.0", "@prisma/adapter-pg": "7.8.0", @@ -178,7 +178,7 @@ }, "packages/dtos": { "name": "@shumai/dtos", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@earendil-works/pi-agent-core": "0.78.0", "zod": "4.4.3", @@ -186,7 +186,7 @@ }, "packages/e2e": { "name": "@shumai/e2e", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@earendil-works/pi-agent-core": "0.78.0", "@google/genai": "1.52.0", @@ -196,15 +196,16 @@ "@shumai/dtos": "workspace:*", "@shumai/transcode": "workspace:*", "@shumai/workflow-core": "workspace:*", + "sharp": "0.35.3", }, }, "packages/tableprinter": { "name": "@shumai/tableprinter", - "version": "0.2.0", + "version": "0.2.2", }, "packages/transcode": { "name": "@shumai/transcode", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@shumai/core": "workspace:*", "@shumai/db": "workspace:*", @@ -218,7 +219,7 @@ }, "packages/webui": { "name": "@shumai/webui", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@base-ui/react": "1.5.0", "@dnd-kit/abstract": "0.4.0", @@ -316,7 +317,7 @@ }, "packages/workflow-core": { "name": "@shumai/workflow-core", - "version": "0.2.0", + "version": "0.2.2", "dependencies": { "@shumai/core": "workspace:*", "@shumai/db": "workspace:*", @@ -581,6 +582,8 @@ "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + "@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.3", "", { "dependencies": { "@img/sharp-wasm32": "0.35.3" }, "os": "freebsd" }, "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg=="], + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], @@ -619,6 +622,8 @@ "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + "@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.3", "", { "dependencies": { "@img/sharp-wasm32": "0.35.3" }, "cpu": "none" }, "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q=="], + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], @@ -2829,6 +2834,10 @@ "@grpc/grpc-js/@grpc/proto-loader": ["@grpc/proto-loader@0.8.1", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg=="], + "@img/sharp-freebsd-wasm32/@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.3", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w=="], + + "@img/sharp-webcontainers-wasm32/@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.3", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w=="], + "@inlang/sdk/@sinclair/typebox": ["@sinclair/typebox@0.31.28", "", {}, "sha512-/s55Jujywdw/Jpan+vsy6JZs1z2ZTGxTmbZTPiuSL2wz9mfzA2gN1zzaqmvfi4pq+uOt7Du85fkiwv5ymW84aQ=="], "@inlang/sdk/uuid": ["uuid@14.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg=="], @@ -3039,6 +3048,8 @@ "@reduxjs/toolkit/immer": ["immer@11.1.8", "", {}, "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA=="], + "@shumai/e2e/sharp": ["sharp@0.35.3", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.5" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.3", "@img/sharp-darwin-x64": "0.35.3", "@img/sharp-freebsd-wasm32": "0.35.3", "@img/sharp-libvips-darwin-arm64": "1.3.2", "@img/sharp-libvips-darwin-x64": "1.3.2", "@img/sharp-libvips-linux-arm": "1.3.2", "@img/sharp-libvips-linux-arm64": "1.3.2", "@img/sharp-libvips-linux-ppc64": "1.3.2", "@img/sharp-libvips-linux-riscv64": "1.3.2", "@img/sharp-libvips-linux-s390x": "1.3.2", "@img/sharp-libvips-linux-x64": "1.3.2", "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", "@img/sharp-libvips-linuxmusl-x64": "1.3.2", "@img/sharp-linux-arm": "0.35.3", "@img/sharp-linux-arm64": "0.35.3", "@img/sharp-linux-ppc64": "0.35.3", "@img/sharp-linux-riscv64": "0.35.3", "@img/sharp-linux-s390x": "0.35.3", "@img/sharp-linux-x64": "0.35.3", "@img/sharp-linuxmusl-arm64": "0.35.3", "@img/sharp-linuxmusl-x64": "0.35.3", "@img/sharp-webcontainers-wasm32": "0.35.3", "@img/sharp-win32-arm64": "0.35.3", "@img/sharp-win32-ia32": "0.35.3", "@img/sharp-win32-x64": "0.35.3" }, "peerDependencies": { "@types/node": "*" }, "optionalPeers": ["@types/node"] }, "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q=="], + "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.10.0", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" }, "bundled": true }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="], @@ -3249,6 +3260,10 @@ "@earendil-works/pi-agent-core/@earendil-works/pi-ai/@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="], + "@img/sharp-freebsd-wasm32/@img/sharp-wasm32/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], + + "@img/sharp-webcontainers-wasm32/@img/sharp-wasm32/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], + "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], "@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], @@ -3357,6 +3372,54 @@ "@radix-ui/react-tooltip/@radix-ui/react-use-controllable-state/@radix-ui/react-use-layout-effect": ["@radix-ui/react-use-layout-effect@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ=="], + "@shumai/e2e/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.2" }, "os": "darwin", "cpu": "arm64" }, "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg=="], + + "@shumai/e2e/sharp/@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.2" }, "os": "darwin", "cpu": "x64" }, "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.2", "", { "os": "linux", "cpu": "arm" }, "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.2", "", { "os": "linux", "cpu": "none" }, "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw=="], + + "@shumai/e2e/sharp/@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ=="], + + "@shumai/e2e/sharp/@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.2" }, "os": "linux", "cpu": "arm" }, "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA=="], + + "@shumai/e2e/sharp/@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.2" }, "os": "linux", "cpu": "arm64" }, "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ=="], + + "@shumai/e2e/sharp/@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.2" }, "os": "linux", "cpu": "ppc64" }, "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA=="], + + "@shumai/e2e/sharp/@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.2" }, "os": "linux", "cpu": "none" }, "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ=="], + + "@shumai/e2e/sharp/@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.2" }, "os": "linux", "cpu": "s390x" }, "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw=="], + + "@shumai/e2e/sharp/@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.2" }, "os": "linux", "cpu": "x64" }, "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA=="], + + "@shumai/e2e/sharp/@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" }, "os": "linux", "cpu": "arm64" }, "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w=="], + + "@shumai/e2e/sharp/@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.2" }, "os": "linux", "cpu": "x64" }, "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg=="], + + "@shumai/e2e/sharp/@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w=="], + + "@shumai/e2e/sharp/@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw=="], + + "@shumai/e2e/sharp/@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.3", "", { "os": "win32", "cpu": "x64" }, "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA=="], + + "@shumai/e2e/sharp/semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + "@tanstack/react-router/@tanstack/react-store/@tanstack/store": ["@tanstack/store@0.9.3", "", {}, "sha512-8reSzl/qGWGGVKhBoxXPMWzATSbZLZFWhwBAFO9NAyp0TxzfBP0mIrGb8CP8KrQTmvzXlR/vFPPUrHTLBGyFyw=="], "@temporalio/proto/protobufjs/@types/node": ["@types/node@22.19.19", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew=="], diff --git a/packages/e2e/package.json b/packages/e2e/package.json index 2595117a..95bec38f 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -4,13 +4,14 @@ "private": true, "type": "module", "dependencies": { - "@shumai/db": "workspace:*", - "@shumai/core": "workspace:*", - "@shumai/workflow-core": "workspace:*", - "@shumai/agent": "workspace:*", - "@shumai/transcode": "workspace:*", - "@shumai/dtos": "workspace:*", "@earendil-works/pi-agent-core": "0.78.0", - "@google/genai": "1.52.0" + "@google/genai": "1.52.0", + "@shumai/agent": "workspace:*", + "@shumai/core": "workspace:*", + "@shumai/db": "workspace:*", + "@shumai/dtos": "workspace:*", + "@shumai/transcode": "workspace:*", + "@shumai/workflow-core": "workspace:*", + "sharp": "0.35.3" } } From dc3dbcb73eb045eb71627a0722d240d079eb728f Mon Sep 17 00:00:00 2001 From: yiling Date: Sat, 25 Jul 2026 11:21:45 +0800 Subject: [PATCH 3/3] fix(transcode): remove debug logs, keep clean epsilon bugfix --- packages/agent/src/tools/screenshot.ts | 13 ------- packages/core/src/transcode/transcode.ts | 46 ------------------------ 2 files changed, 59 deletions(-) diff --git a/packages/agent/src/tools/screenshot.ts b/packages/agent/src/tools/screenshot.ts index 6e9896e8..f2030a1f 100644 --- a/packages/agent/src/tools/screenshot.ts +++ b/packages/agent/src/tools/screenshot.ts @@ -2,7 +2,6 @@ import { Type } from '@sinclair/typebox' import { type AgentTool } from '@earendil-works/pi-agent-core' import { type ImageContent } from '@earendil-works/pi-ai' import { prisma, WorkflowTaskType, WorkflowTaskStatus, type User } from '@shumai/db' -import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' import { workflowService } from '@shumai/workflow-core' import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz' @@ -66,18 +65,6 @@ export function createScreenshotTool( } } - logger.info( - { - assetId, - userId, - userCommentId, - commentTimestamp, - hasAnnotations: Boolean(annotations), - params, - }, - '[screenshotTool] Executing screenshot tool call', - ) - // Trigger transcode workflow to take screenshots const task = await prisma.workflowTask.create({ data: { diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts index 9fa5dd6a..44934621 100644 --- a/packages/core/src/transcode/transcode.ts +++ b/packages/core/src/transcode/transcode.ts @@ -1,4 +1,3 @@ -import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' import { stemFromKey } from '@shumai/core/src/utils/filename' import { prisma, WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' @@ -912,20 +911,6 @@ export class TranscodeService { const videoPath = path.join(tmpDir, path.basename(params.assetKey)) try { - logger.info( - { - assetId: params.assetId, - assetKey: params.assetKey, - start: params.start, - end: params.end, - count: params.count, - commentTimestamp: params.commentTimestamp, - hasAnnotations: Boolean(params.annotations && params.annotations.length > 0), - annotationsCount: params.annotations?.length ?? 0, - }, - '[takeScreenshots] Starting screenshot extraction', - ) - // 1. Download video await s3Service.downloadToFile(bucket, params.assetKey, videoPath) @@ -938,11 +923,6 @@ export class TranscodeService { timestamps = Array.from({ length: params.count }, (_, i) => params.start + i * step) } - logger.info( - { initialTimestamps: timestamps, count: params.count }, - '[takeScreenshots] Generated initial sampling timestamps', - ) - // 3. Snap closest timestamp to commentTimestamp if within range (with 100ms tolerance) const commentTimestamp = params.commentTimestamp const EPSILON = 0.1 @@ -959,30 +939,8 @@ export class TranscodeService { closestIdx = i } } - const oldTs = timestamps[closestIdx] timestamps[closestIdx] = commentTimestamp - logger.info( - { - commentTimestamp, - snappedIdx: closestIdx, - replacedTimestamp: oldTs, - newTimestamp: commentTimestamp, - resultingTimestamps: timestamps, - }, - '[takeScreenshots] Snapped commentTimestamp to closest timestamp index', - ) - } else { - logger.warn( - { - commentTimestamp, - start: params.start, - end: params.end, - }, - '[takeScreenshots] commentTimestamp is OUT OF RANGE [start, end], skipping snap', - ) } - } else { - logger.info('[takeScreenshots] No commentTimestamp provided') } const results: Array<{ key: string; timestamp: number }> = [] @@ -1016,10 +974,6 @@ export class TranscodeService { // 5. Overlay annotations if timestamp matches commentTimestamp (within float tolerance) if (isMatch && params.annotations && params.annotations.length > 0) { - logger.info( - { timestamp: t, commentTimestamp, annotationsCount: params.annotations.length }, - '[takeScreenshots] Overlaying annotations onto frame', - ) const shotBuffer = fs.readFileSync(localShotPath) const composited = await this.overlayAnnotationsOnBuffer(shotBuffer, params.annotations) fs.writeFileSync(localShotPath, composited)