Merge pull request #223 from shumaiOne/test/e2e-video-screenshot-annotation

fix(transcode): handle timestamp rounding tolerance for screenshot annotations
This commit is contained in:
Yiling-J
2026-07-25 11:30:18 +08:00
committed by GitHub
5 changed files with 413 additions and 29 deletions
+77 -14
View File
@@ -33,7 +33,7 @@
},
"apps/agent": {
"name": "@shumai/agent-app",
"version": "0.2.0",
"version": "0.2.2",
"bin": {
"shumai-agent": "./src/index.ts",
},
@@ -45,7 +45,7 @@
},
"apps/cli": {
"name": "@shumai/cli",
"version": "0.2.0",
"version": "0.2.2",
"bin": {
"shumai-cli": "./src/index.ts",
},
@@ -60,7 +60,7 @@
},
"apps/transcode": {
"name": "@shumai/transcode-app",
"version": "0.2.0",
"version": "0.2.2",
"bin": {
"shumai-transcode": "./src/index.ts",
},
@@ -72,7 +72,7 @@
},
"apps/web": {
"name": "@shumai/web-app",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@shumai/agent": "workspace:*",
"@shumai/api": "workspace:*",
@@ -92,7 +92,7 @@
},
"packages/agent": {
"name": "@shumai/agent",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@anthropic-ai/sandbox-runtime": "0.0.52",
"@earendil-works/pi-agent-core": "0.78.0",
@@ -117,7 +117,7 @@
},
"packages/api": {
"name": "@shumai/api",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@hono/zod-validator": "0.7.6",
"@shumai/core": "workspace:*",
@@ -131,7 +131,7 @@
},
"packages/core": {
"name": "@shumai/core",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@shumai/db": "workspace:*",
@@ -162,7 +162,7 @@
},
"packages/db": {
"name": "@shumai/db",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@prisma/adapter-pg": "7.8.0",
@@ -178,7 +178,7 @@
},
"packages/dtos": {
"name": "@shumai/dtos",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"zod": "4.4.3",
@@ -186,7 +186,7 @@
},
"packages/e2e": {
"name": "@shumai/e2e",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@earendil-works/pi-agent-core": "0.78.0",
"@google/genai": "1.52.0",
@@ -196,15 +196,16 @@
"@shumai/dtos": "workspace:*",
"@shumai/transcode": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"sharp": "0.35.3",
},
},
"packages/tableprinter": {
"name": "@shumai/tableprinter",
"version": "0.2.0",
"version": "0.2.2",
},
"packages/transcode": {
"name": "@shumai/transcode",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -218,7 +219,7 @@
},
"packages/webui": {
"name": "@shumai/webui",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@base-ui/react": "1.5.0",
"@dnd-kit/abstract": "0.4.0",
@@ -316,7 +317,7 @@
},
"packages/workflow-core": {
"name": "@shumai/workflow-core",
"version": "0.2.0",
"version": "0.2.2",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -581,6 +582,8 @@
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
"@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.3", "", { "dependencies": { "@img/sharp-wasm32": "0.35.3" }, "os": "freebsd" }, "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg=="],
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
@@ -619,6 +622,8 @@
"@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="],
"@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.3", "", { "dependencies": { "@img/sharp-wasm32": "0.35.3" }, "cpu": "none" }, "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q=="],
"@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
"@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="],
@@ -2829,6 +2834,10 @@
"@grpc/grpc-js/@grpc/proto-loader": ["@grpc/proto-loader@0.8.1", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg=="],
"@img/sharp-freebsd-wasm32/@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.3", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w=="],
"@img/sharp-webcontainers-wasm32/@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.3", "", { "dependencies": { "@emnapi/runtime": "^1.11.1" } }, "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w=="],
"@inlang/sdk/@sinclair/typebox": ["@sinclair/typebox@0.31.28", "", {}, "sha512-/s55Jujywdw/Jpan+vsy6JZs1z2ZTGxTmbZTPiuSL2wz9mfzA2gN1zzaqmvfi4pq+uOt7Du85fkiwv5ymW84aQ=="],
"@inlang/sdk/uuid": ["uuid@14.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg=="],
@@ -3039,6 +3048,8 @@
"@reduxjs/toolkit/immer": ["immer@11.1.8", "", {}, "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA=="],
"@shumai/e2e/sharp": ["sharp@0.35.3", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.5" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.3", "@img/sharp-darwin-x64": "0.35.3", "@img/sharp-freebsd-wasm32": "0.35.3", "@img/sharp-libvips-darwin-arm64": "1.3.2", "@img/sharp-libvips-darwin-x64": "1.3.2", "@img/sharp-libvips-linux-arm": "1.3.2", "@img/sharp-libvips-linux-arm64": "1.3.2", "@img/sharp-libvips-linux-ppc64": "1.3.2", "@img/sharp-libvips-linux-riscv64": "1.3.2", "@img/sharp-libvips-linux-s390x": "1.3.2", "@img/sharp-libvips-linux-x64": "1.3.2", "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", "@img/sharp-libvips-linuxmusl-x64": "1.3.2", "@img/sharp-linux-arm": "0.35.3", "@img/sharp-linux-arm64": "0.35.3", "@img/sharp-linux-ppc64": "0.35.3", "@img/sharp-linux-riscv64": "0.35.3", "@img/sharp-linux-s390x": "0.35.3", "@img/sharp-linux-x64": "0.35.3", "@img/sharp-linuxmusl-arm64": "0.35.3", "@img/sharp-linuxmusl-x64": "0.35.3", "@img/sharp-webcontainers-wasm32": "0.35.3", "@img/sharp-win32-arm64": "0.35.3", "@img/sharp-win32-ia32": "0.35.3", "@img/sharp-win32-x64": "0.35.3" }, "peerDependencies": { "@types/node": "*" }, "optionalPeers": ["@types/node"] }, "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q=="],
"@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.10.0", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" }, "bundled": true }, "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw=="],
"@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="],
@@ -3249,6 +3260,10 @@
"@earendil-works/pi-agent-core/@earendil-works/pi-ai/@mistralai/mistralai": ["@mistralai/mistralai@2.2.1", "", { "dependencies": { "ws": "^8.18.0", "zod": "^3.25.0 || ^4.0.0", "zod-to-json-schema": "^3.25.0" } }, "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ=="],
"@img/sharp-freebsd-wasm32/@img/sharp-wasm32/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="],
"@img/sharp-webcontainers-wasm32/@img/sharp-wasm32/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="],
"@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
"@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
@@ -3357,6 +3372,54 @@
"@radix-ui/react-tooltip/@radix-ui/react-use-controllable-state/@radix-ui/react-use-layout-effect": ["@radix-ui/react-use-layout-effect@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ=="],
"@shumai/e2e/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.2" }, "os": "darwin", "cpu": "arm64" }, "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg=="],
"@shumai/e2e/sharp/@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.2" }, "os": "darwin", "cpu": "x64" }, "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w=="],
"@shumai/e2e/sharp/@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg=="],
"@shumai/e2e/sharp/@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.2", "", { "os": "linux", "cpu": "arm" }, "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.2", "", { "os": "linux", "cpu": "none" }, "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw=="],
"@shumai/e2e/sharp/@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.2", "", { "os": "linux", "cpu": "x64" }, "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ=="],
"@shumai/e2e/sharp/@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.2" }, "os": "linux", "cpu": "arm" }, "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA=="],
"@shumai/e2e/sharp/@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.2" }, "os": "linux", "cpu": "arm64" }, "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ=="],
"@shumai/e2e/sharp/@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.2" }, "os": "linux", "cpu": "ppc64" }, "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA=="],
"@shumai/e2e/sharp/@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.2" }, "os": "linux", "cpu": "none" }, "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ=="],
"@shumai/e2e/sharp/@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.2" }, "os": "linux", "cpu": "s390x" }, "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw=="],
"@shumai/e2e/sharp/@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.2" }, "os": "linux", "cpu": "x64" }, "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA=="],
"@shumai/e2e/sharp/@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" }, "os": "linux", "cpu": "arm64" }, "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w=="],
"@shumai/e2e/sharp/@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.3", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.2" }, "os": "linux", "cpu": "x64" }, "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg=="],
"@shumai/e2e/sharp/@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w=="],
"@shumai/e2e/sharp/@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.3", "", { "os": "win32", "cpu": "ia32" }, "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw=="],
"@shumai/e2e/sharp/@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.3", "", { "os": "win32", "cpu": "x64" }, "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA=="],
"@shumai/e2e/sharp/semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="],
"@tanstack/react-router/@tanstack/react-store/@tanstack/store": ["@tanstack/store@0.9.3", "", {}, "sha512-8reSzl/qGWGGVKhBoxXPMWzATSbZLZFWhwBAFO9NAyp0TxzfBP0mIrGb8CP8KrQTmvzXlR/vFPPUrHTLBGyFyw=="],
"@temporalio/proto/protobufjs/@types/node": ["@types/node@22.19.19", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew=="],
@@ -646,6 +646,97 @@ describe('TranscodeService', () => {
expect(b).toBeLessThan(100)
})
it('should snap commentTimestamp and overlay annotations when start/end has rounding mismatch', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(file: string, args: string[], cb: any) => {
if (file === 'ffmpeg') {
const outPath = args[args.length - 1]
fs.writeFileSync(outPath, 'fake-webp-image')
}
cb(null, { stdout: '', stderr: '' })
},
)
const overlaySpy = vi
.spyOn(transcodeService, 'overlayAnnotationsOnBuffer')
.mockResolvedValue(Buffer.from('composited-buffer'))
const annotations: PrismaJson.AnnotationList = [
{
type: 'box',
color: '#ff0000',
points: [
[0.2, 0.2],
[0.8, 0.8],
],
},
]
const results = await transcodeService.takeScreenshots({
assetKey: 'test/video.mp4',
assetId: 'asset-123',
start: 4.57,
end: 4.57,
count: 1,
commentTimestamp: 4.566666666666667,
annotations,
})
expect(results).toHaveLength(1)
expect(results[0].timestamp).toBe(4.566666666666667)
expect(overlaySpy).toHaveBeenCalledWith(expect.any(Buffer), annotations)
overlaySpy.mockRestore()
})
it('should overlay annotation on ONLY the single snapped frame in a multi-screenshot range', async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(file: string, args: string[], cb: any) => {
if (file === 'ffmpeg') {
const outPath = args[args.length - 1]
fs.writeFileSync(outPath, 'fake-webp-image')
}
cb(null, { stdout: '', stderr: '' })
},
)
const overlaySpy = vi
.spyOn(transcodeService, 'overlayAnnotationsOnBuffer')
.mockResolvedValue(Buffer.from('composited-buffer'))
const annotations: PrismaJson.AnnotationList = [
{
type: 'box',
color: '#ff0000',
points: [
[0.2, 0.2],
[0.8, 0.8],
],
},
]
const results = await transcodeService.takeScreenshots({
assetKey: 'test/video.mp4',
assetId: 'asset-123',
start: 0,
end: 1,
count: 30,
commentTimestamp: 0.566666666666667,
annotations,
})
expect(results).toHaveLength(30)
// Exactly 1 overlay call for the snapped timestamp out of 30 frames
expect(overlaySpy).toHaveBeenCalledTimes(1)
expect(overlaySpy).toHaveBeenCalledWith(expect.any(Buffer), annotations)
overlaySpy.mockRestore()
})
it('should generate PDF from text file including CJK characters', async () => {
const txtFile = path.join(tempDir, 'test.txt')
const pdfFile = path.join(tempDir, 'output.pdf')
+10 -8
View File
@@ -923,10 +923,13 @@ export class TranscodeService {
timestamps = Array.from({ length: params.count }, (_, i) => params.start + i * step)
}
// 3. Snap closest timestamp to commentTimestamp if within range
// 3. Snap closest timestamp to commentTimestamp if within range (with 100ms tolerance)
const commentTimestamp = params.commentTimestamp
const EPSILON = 0.1
if (commentTimestamp !== undefined && commentTimestamp !== null) {
if (commentTimestamp >= params.start && commentTimestamp <= params.end) {
const inRange =
commentTimestamp >= params.start - EPSILON && commentTimestamp <= params.end + EPSILON
if (inRange) {
let closestIdx = 0
let minDiff = Math.abs(timestamps[0] - commentTimestamp)
for (let i = 1; i < timestamps.length; i++) {
@@ -964,14 +967,13 @@ export class TranscodeService {
]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
// 5. Overlay annotations if timestamp matches commentTimestamp exactly
if (
const isMatch =
commentTimestamp !== undefined &&
commentTimestamp !== null &&
t === commentTimestamp &&
params.annotations &&
params.annotations.length > 0
) {
Math.abs(t - commentTimestamp) < 1e-4
// 5. Overlay annotations if timestamp matches commentTimestamp (within float tolerance)
if (isMatch && params.annotations && params.annotations.length > 0) {
const shotBuffer = fs.readFileSync(localShotPath)
const composited = await this.overlayAnnotationsOnBuffer(shotBuffer, params.annotations)
fs.writeFileSync(localShotPath, composited)
+8 -7
View File
@@ -4,13 +4,14 @@
"private": true,
"type": "module",
"dependencies": {
"@shumai/db": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"@shumai/agent": "workspace:*",
"@shumai/transcode": "workspace:*",
"@shumai/dtos": "workspace:*",
"@earendil-works/pi-agent-core": "0.78.0",
"@google/genai": "1.52.0"
"@google/genai": "1.52.0",
"@shumai/agent": "workspace:*",
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
"@shumai/dtos": "workspace:*",
"@shumai/transcode": "workspace:*",
"@shumai/workflow-core": "workspace:*",
"sharp": "0.35.3"
}
}
@@ -7,6 +7,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { fileURLToPath } from 'url'
import * as path from 'path'
import * as fs from 'fs'
import sharp from 'sharp'
const currentDir = path.dirname(fileURLToPath(import.meta.url))
const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts')
@@ -126,5 +127,231 @@ describe.each(['local', 'temporal'] as const)(
expect(output.screenshots.length).toBe(1)
expect(output.screenshots[0].key).toContain('screenshots/')
}, 50000)
it('should draw annotation box correctly for 5 random comment timestamps', async () => {
// 1. Seed Database
const team = await prisma.team.create({
data: { name: 'E2E Video Screenshot Annotation Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Video Screenshot Annotation Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: {
key: 'projects/e2e/video-shot-ann.mp4',
},
})
const asset = await prisma.asset.create({
data: {
name: 'video-shot-ann.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
// 2. Seed S3 Storage from Fixture
const mp4Path = path.join(fixturesDir, 'small.mp4')
const mp4Buffer = fs.readFileSync(mp4Path)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/video-shot-ann.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
// 3. Generate 5 random comment timestamps between 0.05s and 0.95s
const randomTimestamps: number[] = []
for (let i = 0; i < 5; i++) {
const randomTs = 0.05 + Math.random() * 0.9
randomTimestamps.push(randomTs)
}
console.log('Testing random timestamps:', randomTimestamps)
const boxAnnotation = [
{
type: 'box' as const,
color: '#ff0000',
points: [
[0.2, 0.2],
[0.8, 0.8],
],
},
]
for (const commentTimestamp of randomTimestamps) {
// Create Workflow Task with commentTimestamp & annotations
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_screenshot',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
screenshot: {
start: 0,
end: 1,
count: 5,
commentTimestamp,
annotations: boxAnnotation,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
const output = completedTask.output as unknown as {
screenshots: { key: string; timestamp: number }[]
}
expect(output).toBeDefined()
expect(output.screenshots).toBeDefined()
// Find the screenshot matching commentTimestamp
const matchingShot = output.screenshots.find(
(s) => Math.abs(s.timestamp - commentTimestamp) < 1e-6,
)
expect(matchingShot).toBeDefined()
// Download screenshot buffer from S3
const s3Obj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
matchingShot!.key,
)
const shotBuffer = s3Obj.buffer
// Verify pixel data using sharp to check if red box annotation was overlaid
const { data, info } = await sharp(shotBuffer).raw().toBuffer({ resolveWithObject: true })
// Target pixel inside box (e.g. x = 50%, y = 20% near top edge of red box)
const pixelX = Math.floor(info.width * 0.5)
const pixelY = Math.floor(info.height * 0.2)
const offset = (pixelY * info.width + pixelX) * info.channels
const r = data[offset]
const g = data[offset + 1]
const b = data[offset + 2]
// Red box color #ff0000 should result in high red channel and low green/blue channels
expect(r).toBeGreaterThan(150)
expect(g).toBeLessThan(120)
expect(b).toBeLessThan(120)
}
}, 90000)
it('should overlay annotation when commentTimestamp has rounding mismatch with start/end', async () => {
// 1. Seed Database
const team = await prisma.team.create({
data: { name: 'E2E Video Screenshot Rounding Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Video Screenshot Rounding Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: {
key: 'projects/e2e/video-shot-rounding.mp4',
},
})
const asset = await prisma.asset.create({
data: {
name: 'video-shot-rounding.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
// 2. Seed S3 Storage from Fixture
const mp4Path = path.join(fixturesDir, 'small.mp4')
const mp4Buffer = fs.readFileSync(mp4Path)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/video-shot-rounding.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
// Reproduce log scenario: exact timestamp is 0.566666666666667, Agent requested rounded 0.57
const commentTimestamp = 0.566666666666667
const start = 0.57
const end = 0.57
const boxAnnotation = [
{
type: 'box' as const,
color: '#ff0000',
points: [
[0.2, 0.2],
[0.8, 0.8],
],
},
]
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_screenshot',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
screenshot: {
start,
end,
count: 1,
commentTimestamp,
annotations: boxAnnotation,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
const output = completedTask.output as unknown as {
screenshots: { key: string; timestamp: number }[]
}
expect(output).toBeDefined()
expect(output.screenshots).toBeDefined()
expect(output.screenshots.length).toBe(1)
const s3Obj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
output.screenshots[0].key,
)
const shotBuffer = s3Obj.buffer
const { data, info } = await sharp(shotBuffer).raw().toBuffer({ resolveWithObject: true })
const pixelX = Math.floor(info.width * 0.5)
const pixelY = Math.floor(info.height * 0.2)
const offset = (pixelY * info.width + pixelX) * info.channels
const r = data[offset]
const g = data[offset + 1]
const b = data[offset + 2]
expect(r).toBeGreaterThan(150)
expect(g).toBeLessThan(120)
expect(b).toBeLessThan(120)
}, 50000)
},
)