Compare commits

...
10 Commits
Author SHA1 Message Date
yiling f8beb195dd Release v0.5.1 2026-10-06 22:29:50 +08:00
yiling a97996cb59 docs(changelog): note clickable notification cards 2026-10-06 22:18:18 +08:00
jorgeandYiling-J f870ca9b49 feat(asset): link to assets on notification cards. (#451)
* feat(asset): link to assets on notification cards.

* Update packages/webui/components/notification-card.tsx

Co-authored-by: Yiling-J <njjyl723@gmail.com>

---------

Co-authored-by: Yiling-J <njjyl723@gmail.com>
2026-10-06 22:14:11 +08:00
Yiling-J 2e5ff281a8 feat(transcode): add hardware decoding support for nvenc, qsv, and rkmpp (#452)
* feat(transcode): add hardware decoding support for nvenc, qsv, and rkmpp

* refactor(transcode): eliminate redundant hardware decode gating and dead condition
2026-10-06 17:35:14 +08:00
yiling de8decf113 fix(ci): prevent tagging main on PR workflow_dispatch docker builds 2026-10-06 11:49:49 +08:00
Cameron HarrisandClaude Opus 5.5 dd57fb9ed2 feat(transcode): decode and scale on the GPU for hardware transcodes (VA-API) (#449)
* feat(transcode): decode and scale on the GPU for VA-API transcodes

When the selected encoder is h264_vaapi, decode with -hwaccel vaapi and
scale with scale_vaapi so frames stay in GPU memory, instead of decoding
and scaling on the CPU and hwupload-ing before the encoder. On a Haswell
iGPU this makes 4K proxy/HLS renditions 3-5x faster.

Fallback order: GPU decode + VA-API encode -> software decode + VA-API
encode (previous behavior) -> libx264. Watermark overlays, HDR sources
and rotated sources skip GPU decode: ffmpeg cannot auto-insert its
rotation filter on hardware frames, so rotated input would otherwise come
out sideways instead of failing. SHUMAI_VAAPI_HW_DECODE=false disables it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(transcode): document VA-API GPU decoding and SHUMAI_VAAPI_HW_DECODE

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(transcode): generic hardware decode config, stall watchdog, reuse probed rotation

Follow-ups from comparing with Immich and Jellyfin:

- Replace the VA-API-specific helpers with a per-encoder HW_DECODE_CONFIGS
  table (input args, output args, GPU scale filter) and build the MP4 and
  HLS input/filter graph in one shared method so the paths cannot drift.
  Only h264_vaapi has an entry; other backends can be added once tested on
  real hardware. The opt-out is now SHUMAI_HW_DECODE=false.
- Kill a GPU-decode attempt with SIGKILL when its output stops growing for
  SHUMAI_HW_DECODE_STALL_TIMEOUT (default 120s) and fall back, since some
  VA-API driver failures hang ffmpeg instead of exiting (immich#31933).
- Add -noautoscale on the GPU-decode path so a mid-stream decoder reinit
  (e.g. 1080 -> 1088) cannot auto-insert a software scaler into the GPU
  graph (jellyfin#17133).
- Pass the rotation already probed into MediaMetadata through the workflow
  and activities as sourceRotation; ffprobe only runs when it is missing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-05 12:00:01 +08:00
Yiling-J 9a640632aa ci(release): allow manual Docker image build from external PR (#450) 2026-10-05 09:48:48 +08:00
yiling e61641d7ec fix(workflow): increase activity startToCloseTimeout to 120 minutes 2026-10-04 10:22:31 +08:00
Yiling-J 7ca8ba8b2a fix(transcode): resolve RAW transcoding hang in Docker deployments (#447) 2026-10-03 12:45:12 +08:00
Yiling-J 8805d5799a feat(transcode): add dcraw_emu fallback for RAW image preview generation (#446) 2026-10-03 11:03:31 +08:00
36 changed files with 2138 additions and 184 deletions
+11 -1
View File
@@ -10,6 +10,10 @@ on:
description: 'Docker image tag (e.g. feature-vaapi)'
required: false
type: string
pr:
description: 'Pull request number (e.g. 449)'
required: false
type: string
jobs:
build-and-push:
@@ -20,7 +24,10 @@ jobs:
steps:
- name: Checkout repository
id: checkout
uses: actions/checkout@v4
with:
ref: ${{ inputs.pr && format('refs/pull/{0}/head', inputs.pr) || github.ref }}
# Set up Docker Buildx
- name: Set up Docker Buildx
@@ -51,8 +58,11 @@ jobs:
crpi-000nkafzu6xvpwsc.cn-hongkong.personal.cr.aliyuncs.com/shumai-one/shumai
tags: |
type=semver,pattern={{version}}
type=ref,event=branch
type=ref,event=branch,enable=${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.pr && !inputs.tag) }}
type=raw,value=pr-${{ inputs.pr }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.pr != '' && inputs.tag == '' }}
type=raw,value=${{ inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' && inputs.tag != '' }}
labels: |
org.opencontainers.image.revision=${{ steps.checkout.outputs.commit }}
# Build and push Docker image
- name: Build and push Docker image
+13
View File
@@ -12,6 +12,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Changed
## [0.5.1] - 2026-10-06
### Added
* **notification**: Notification cards are now clickable, taking you directly to the relevant file
* **transcode**: Add hardware decoding and GPU scaling support for NVIDIA NVENC, Intel QuickSync Video (QSV), Rockchip (RKMPP), and VA-API (Intel and AMD GPUs), significantly speeding up video proxy and HLS rendition generation across supported platforms. Set `SHUMAI_HW_DECODE=false` to turn hardware decoding off
* **transcode**: Add `dcraw_emu` fallback to generate WebP previews for camera RAW files without embedded JPEG previews
### Fixed
### Changed
## [0.5.0] - 2026-10-02
### Added
+2
View File
@@ -52,6 +52,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
poppler-utils \
imagemagick \
fonts-noto-cjk \
libraw-bin \
libimage-exiftool-perl \
&& . /etc/os-release \
&& TARGETARCH=$(dpkg --print-architecture) \
&& FFMPEG_VERSION="8.1.2-5" \
+6 -4
View File
@@ -162,23 +162,24 @@ Shumai requires the following system packages on Linux:
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
- **`poppler`** (`poppler-utils`) – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
- **`imagemagick`** – Used for PSD format transcoding and color profile conversion to sRGB.
- **`dcraw_emu`** (`libraw` / `libraw-bin`) – Used for camera RAW format image decoding and preview generation.
- **`bubblewrap`**, **`socat`**, and **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`) for process isolation, networking, and workspace search.
Install all required packages with one command:
> Ubuntu/Debian
```bash
sudo apt install -y ffmpeg poppler-utils imagemagick bubblewrap socat ripgrep
sudo apt install -y ffmpeg poppler-utils imagemagick bubblewrap socat ripgrep libraw-bin
```
> Fedora
```bash
sudo dnf install -y ffmpeg poppler-utils ImageMagick bubblewrap socat ripgrep
sudo dnf install -y ffmpeg poppler-utils ImageMagick bubblewrap socat ripgrep LibRaw
```
> Arch Linux
```bash
sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep
sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep libraw
```
> [!NOTE]
@@ -201,12 +202,13 @@ Shumai requires:
- **`ffmpeg`** – Used for media transcoding and metadata extraction.
- **`poppler`** – Used for PDF page image extraction and PDF sprite preview generation (`pdftoppm`).
- **`imagemagick`** – Used for PSD format transcoding and color profile conversion to sRGB.
- **`dcraw_emu`** (`libraw`) – Used for camera RAW format image decoding and preview generation.
- **`ripgrep`** – Required by the AI agent sandbox (`anthropic-experimental/sandbox-runtime`).
Install the required packages with Homebrew:
```bash
brew install ffmpeg poppler imagemagick ripgrep
brew install ffmpeg poppler imagemagick ripgrep libraw
```
---
+7 -5
View File
@@ -187,23 +187,24 @@ mkdir shumai && cd shumai
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
- **`poppler`** (`poppler-utils`) —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
- **`imagemagick`** —— 用于 PSD 格式图像转码和 sRGB 色彩空间转换。
- **`dcraw_emu`** (`libraw` / `libraw-bin`) —— 用于相机 RAW 格式图像解码和预览图生成。
- **`bubblewrap`**、**`socat`** 和 **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需,用于进程隔离、网络通信和工作区搜索。
可以使用以下命令一次性安装所有依赖:
> Ubuntu/Debian
```bash
sudo apt install -y ffmpeg poppler-utils imagemagick bubblewrap socat ripgrep
sudo apt install -y ffmpeg poppler-utils imagemagick bubblewrap socat ripgrep libraw-bin
```
> Fedora
```bash
sudo dnf install -y ffmpeg poppler-utils ImageMagick bubblewrap socat ripgrep
sudo dnf install -y ffmpeg poppler-utils ImageMagick bubblewrap socat ripgrep LibRaw
```
> Arch Linux
```bash
sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep
sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep libraw
```
> [!NOTE]
@@ -215,7 +216,7 @@ sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep
> sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
> ```
>
> 或者,配置 AppArmor Profile,为相关可执行文件授予所需的 `userns` 权限。
> 或者,配置 AppArmor Profile,为相关可执行文件授予所需的 `userns`权限。
---
@@ -226,12 +227,13 @@ sudo pacman -S --noconfirm ffmpeg poppler imagemagick bubblewrap socat ripgrep
- **`ffmpeg`** —— 用于媒体转码和元数据提取。
- **`poppler`** —— 用于 PDF 页面图像提取和 PDF 雪碧图预览生成 (`pdftoppm`)。
- **`imagemagick`** —— 用于 PSD 格式图像转码和 sRGB 色彩空间转换。
- **`dcraw_emu`** (`libraw`) —— 用于相机 RAW 格式图像解码和预览图生成。
- **`ripgrep`** —— AI Agent 沙箱(`anthropic-experimental/sandbox-runtime`)所需。
使用 Homebrew 安装所需依赖:
```bash
brew install ffmpeg poppler imagemagick ripgrep
brew install ffmpeg poppler imagemagick ripgrep libraw
```
---
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/agent-app",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/cli",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/premiere-uxp",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "commonjs",
"scripts": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/transcode-app",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"bin": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/web-app",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"dependencies": {
+15 -15
View File
@@ -33,7 +33,7 @@
},
"apps/agent": {
"name": "@shumai/agent-app",
"version": "0.5.0",
"version": "0.5.1",
"bin": {
"shumai-agent": "./src/index.ts",
},
@@ -45,7 +45,7 @@
},
"apps/cli": {
"name": "@shumai/cli",
"version": "0.5.0",
"version": "0.5.1",
"bin": {
"shumai-cli": "./src/index.ts",
},
@@ -60,7 +60,7 @@
},
"apps/premiere-uxp": {
"name": "@shumai/premiere-uxp",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@shumai/api": "workspace:*",
"@shumai/dtos": "workspace:*",
@@ -114,7 +114,7 @@
},
"apps/transcode": {
"name": "@shumai/transcode-app",
"version": "0.5.0",
"version": "0.5.1",
"bin": {
"shumai-transcode": "./src/index.ts",
},
@@ -126,7 +126,7 @@
},
"apps/web": {
"name": "@shumai/web-app",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@shumai/agent": "workspace:*",
"@shumai/api": "workspace:*",
@@ -146,7 +146,7 @@
},
"packages/agent": {
"name": "@shumai/agent",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@anthropic-ai/sandbox-runtime": "0.0.74",
"@earendil-works/pi-agent-core": "0.82.1",
@@ -171,7 +171,7 @@
},
"packages/api": {
"name": "@shumai/api",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@hono/zod-validator": "0.7.6",
"@shumai/core": "workspace:*",
@@ -185,7 +185,7 @@
},
"packages/core": {
"name": "@shumai/core",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@ai-sdk/alibaba": "2.0.38",
"@ai-sdk/amazon-bedrock": "5.0.67",
@@ -247,7 +247,7 @@
},
"packages/db": {
"name": "@shumai/db",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"@prisma/adapter-pg": "7.10.0",
@@ -263,7 +263,7 @@
},
"packages/dtos": {
"name": "@shumai/dtos",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"zod": "4.5.4",
@@ -271,7 +271,7 @@
},
"packages/e2e": {
"name": "@shumai/e2e",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@earendil-works/pi-agent-core": "0.82.1",
"@google/genai": "2.19.0",
@@ -286,11 +286,11 @@
},
"packages/tableprinter": {
"name": "@shumai/tableprinter",
"version": "0.5.0",
"version": "0.5.1",
},
"packages/transcode": {
"name": "@shumai/transcode",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
@@ -304,7 +304,7 @@
},
"packages/webui": {
"name": "@shumai/webui",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@base-ui/react": "1.7.0",
"@dnd-kit/abstract": "0.5.0",
@@ -406,7 +406,7 @@
},
"packages/workflow-core": {
"name": "@shumai/workflow-core",
"version": "0.5.0",
"version": "0.5.1",
"dependencies": {
"@shumai/core": "workspace:*",
"@shumai/db": "workspace:*",
+14
View File
@@ -3,6 +3,20 @@ title: "Changelog"
description: "Recent updates, new features, and improvements in Shumai"
---
<Update label="v0.5.1" description="October 6, 2026">
### Added
* **notification**: Notification cards are now clickable, taking you directly to the relevant file
* **transcode**: Add hardware decoding and GPU scaling support for NVIDIA NVENC, Intel QuickSync Video (QSV), Rockchip (RKMPP), and VA-API (Intel and AMD GPUs), significantly speeding up video proxy and HLS rendition generation across supported platforms. Set `SHUMAI_HW_DECODE=false` to turn hardware decoding off
* **transcode**: Add `dcraw_emu` fallback to generate WebP previews for camera RAW files without embedded JPEG previews
### Fixed
### Changed
</Update>
<Update label="v0.5.0" description="October 2, 2026">
### Added
@@ -55,6 +55,20 @@ Shumai includes a built-in usability probe that tests hardware encoders at start
You should see `card0` and `renderD128` (or similar).
2. Shumai's Docker image automatically adjusts group permissions on container startup so the non-root application user can access host `/dev/dri` devices without permission errors.
#### GPU decoding
With VA-API, Shumai decodes and resizes video on the GPU as well as encoding it, so frames never leave GPU memory. This is much faster than decoding on the CPU, especially for 4K sources on integrated graphics. (Other backends currently decode on the CPU and encode on the GPU.)
Some videos automatically use CPU decoding with GPU encoding instead:
- Codecs your GPU cannot decode, such as HEVC or 10-bit video on older Intel chips. Shumai tries the GPU first and retries on the CPU within a second.
- Rotated videos, such as portrait phone recordings, so they keep the correct orientation.
- Watermarked share links and HDR sources.
If a GPU-decode attempt stops making progress for 2 minutes (some driver bugs make ffmpeg hang instead of failing), Shumai stops it and retries on the CPU. Change the limit with `SHUMAI_HW_DECODE_STALL_TIMEOUT` (in seconds).
To always decode on the CPU, set `SHUMAI_HW_DECODE=false` on the `shumai` service.
### WSL2 (Windows)
- Ensure you are running Windows 11 with WSL2 and have installed the appropriate GPU drivers on the Windows host.
+3
View File
@@ -44,6 +44,9 @@ To handle media processing and security isolation, the host machine must have th
- **FFmpeg & FFprobe**: Required for video transcoding and metadata extraction.
- _macOS_: `brew install ffmpeg`
- _Linux (Ubuntu/Debian)_: `sudo apt-get install ffmpeg`
- **dcraw_emu (LibRaw)**: Required for camera RAW image decoding and preview generation.
- _macOS_: `brew install libraw`
- _Linux (Ubuntu/Debian)_: `sudo apt-get install libraw-bin`
- **Agent Sandbox Utilities**: Used to run custom automation scripts securely.
- _macOS_: `brew install ripgrep`
- _Linux (Ubuntu/Debian)_: `sudo apt-get install bubblewrap socat ripgrep`
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "shumai",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"repository": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/agent",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/api",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/core",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+465 -11
View File
@@ -3,8 +3,11 @@ import * as fs from 'fs'
import {
extractEmbeddedJpeg,
validateExtractedJpeg,
validateExtractedImage,
decodeRawWithDcraw,
extractAndValidateRawPreview,
EXIF_ORIENTATION_TO_ROTATION,
withTimeout,
} from './raw-extract'
// Mock exiftool-vendored
@@ -26,13 +29,20 @@ vi.mock('sharp', () => {
return { default: mockSharp }
})
// Mock child_process
vi.mock('child_process', () => ({
execFile: vi.fn(),
}))
import { exiftool } from 'exiftool-vendored'
import sharp from 'sharp'
import { execFile } from 'child_process'
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const mockMetadata = (sharp as any)._mockMetadata as ReturnType<typeof vi.fn>
const mockExtractBinaryTag = vi.mocked(exiftool.extractBinaryTag)
const mockRead = vi.mocked(exiftool.read)
const mockExecFile = vi.mocked(execFile)
describe('extractEmbeddedJpeg', () => {
beforeEach(() => {
@@ -162,6 +172,29 @@ describe('extractEmbeddedJpeg', () => {
expect(result!.orientation).toBeUndefined()
result!.cleanup()
})
it('skips remaining tags and aborts early when extraction times out', async () => {
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
try {
process.env.EXIFTOOL_TIMEOUT_MS = '20'
// Hang on first tag
mockExtractBinaryTag.mockImplementationOnce(
() => new Promise((resolve) => setTimeout(resolve, 300)),
)
const result = await extractEmbeddedJpeg('/path/to/photo.cr2')
expect(result).toBeNull()
// Crucial: should break immediately after the first timeout instead of trying remaining 2 tags
expect(mockExtractBinaryTag).toHaveBeenCalledTimes(1)
} finally {
if (origEnv !== undefined) {
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
} else {
delete process.env.EXIFTOOL_TIMEOUT_MS
}
}
})
})
describe('validateExtractedJpeg', () => {
@@ -202,16 +235,265 @@ describe('validateExtractedJpeg', () => {
})
})
describe('validateExtractedImage', () => {
it('is an alias for validateExtractedJpeg', () => {
expect(validateExtractedImage).toBe(validateExtractedJpeg)
})
})
describe('decodeRawWithDcraw', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('decodes RAW using camera white balance, sRGB, and TIFF output without -h when longest dimension <= 8192', async () => {
mockRead.mockResolvedValueOnce({
ImageWidth: 6000,
ImageHeight: 4000,
Orientation: 1,
} as never)
let executedArgs: string[] = []
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
executedArgs = args as string[]
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
expect(result).not.toBeNull()
expect(fs.existsSync(result!.previewPath)).toBe(true)
expect(result!.orientation).toBeUndefined()
expect(result!.rawWidth).toBe(6000)
expect(result!.rawHeight).toBe(4000)
expect(executedArgs).toContain('-w')
expect(executedArgs).toContain('-o')
expect(executedArgs).toContain('1')
expect(executedArgs).toContain('-T')
expect(executedArgs).toContain('-Z')
expect(executedArgs).not.toContain('-h')
expect(executedArgs[executedArgs.length - 1]).toBe('/path/to/photo.cr2')
result!.cleanup()
expect(fs.existsSync(result!.previewPath)).toBe(false)
})
it('enables half-size (-h) when longest dimension > 8192px and computes upright raw dimensions', async () => {
mockRead.mockResolvedValueOnce({
ImageWidth: 9504,
ImageHeight: 6336,
Orientation: 6,
} as never)
let executedArgs: string[] = []
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
executedArgs = args as string[]
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.arw')
expect(result).not.toBeNull()
expect(executedArgs).toContain('-h')
expect(result!.orientation).toBeUndefined()
// For orientation 6 (90 CW), upright raw dimensions are swapped
expect(result!.rawWidth).toBe(6336)
expect(result!.rawHeight).toBe(9504)
result!.cleanup()
})
it('omits -h when longest dimension is exactly 8192px', async () => {
mockRead.mockResolvedValueOnce({
ImageWidth: 8192,
ImageHeight: 5464,
Orientation: 1,
} as never)
let executedArgs: string[] = []
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
executedArgs = args as string[]
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.nef')
expect(result).not.toBeNull()
expect(executedArgs).not.toContain('-h')
result!.cleanup()
})
it('defaults to full size without -h when EXIF metadata cannot be read', async () => {
mockRead.mockRejectedValueOnce(new Error('Exif read failure'))
let executedArgs: string[] = []
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
executedArgs = args as string[]
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.dng')
expect(result).not.toBeNull()
expect(executedArgs).not.toContain('-h')
result!.cleanup()
})
it('defaults to full size without -h when EXIF metadata read times out', async () => {
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
try {
process.env.EXIFTOOL_TIMEOUT_MS = '20'
mockRead.mockImplementationOnce(() => new Promise((resolve) => setTimeout(resolve, 300)))
let executedArgs: string[] = []
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
executedArgs = args as string[]
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.dng')
expect(result).not.toBeNull()
expect(executedArgs).not.toContain('-h')
result!.cleanup()
} finally {
if (origEnv !== undefined) {
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
} else {
delete process.env.EXIFTOOL_TIMEOUT_MS
}
}
})
it('returns null and cleans up when dcraw_emu exits with non-zero code', async () => {
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
const callback = typeof _opts === 'function' ? _opts : cb
const err = new Error('Command failed: dcraw_emu')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(err as any).code = 1
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(err)
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
expect(result).toBeNull()
})
it('returns null and cleans up when dcraw_emu times out', async () => {
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
const callback = typeof _opts === 'function' ? _opts : cb
const err = new Error('Process timed out')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(err as any).code = 'ETIMEDOUT'
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(err)
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
expect(result).toBeNull()
})
it('returns null and cleans up when dcraw_emu is not found (ENOENT)', async () => {
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
const callback = typeof _opts === 'function' ? _opts : cb
const err = new Error('spawn dcraw_emu ENOENT')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(err as any).code = 'ENOENT'
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(err)
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
expect(result).toBeNull()
})
it('returns null and cleans up when output TIFF is missing or empty', async () => {
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await decodeRawWithDcraw('/path/to/photo.cr2')
expect(result).toBeNull()
})
})
describe('extractAndValidateRawPreview', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('returns previewPath, dimensions, orientation, and cleanup for valid RAW', async () => {
it('returns previewPath, dimensions, orientation, and cleanup for valid RAW using embedded JPEG first', async () => {
mockExtractBinaryTag.mockImplementationOnce(async (_tag, _src, dest) => {
fs.writeFileSync(dest, 'jpeg-data')
})
mockRead.mockResolvedValueOnce({ Orientation: 6 } as never)
mockRead.mockResolvedValueOnce({ Orientation: 6, ImageWidth: 4000, ImageHeight: 3000 } as never)
mockMetadata.mockResolvedValueOnce({ width: 4000, height: 3000 })
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
@@ -221,35 +503,207 @@ describe('extractAndValidateRawPreview', () => {
expect(result!.width).toBe(4000)
expect(result!.height).toBe(3000)
expect(result!.orientation).toBe(6)
expect(mockExecFile).not.toHaveBeenCalled()
result!.cleanup()
expect(fs.existsSync(result!.previewPath)).toBe(false)
})
it('returns null when extraction fails', async () => {
it('falls back to dcraw_emu when embedded preview extraction fails', async () => {
mockExtractBinaryTag
.mockRejectedValueOnce(new Error('fail'))
.mockRejectedValueOnce(new Error('fail'))
.mockRejectedValueOnce(new Error('fail'))
mockRead.mockResolvedValueOnce({
ImageWidth: 6000,
ImageHeight: 4000,
Orientation: 1,
} as never)
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
expect(result).not.toBeNull()
expect(mockExecFile).toHaveBeenCalled()
expect(result!.width).toBe(6000)
expect(result!.height).toBe(4000)
expect(result!.rawWidth).toBe(6000)
expect(result!.rawHeight).toBe(4000)
expect(result!.orientation).toBeUndefined()
result!.cleanup()
expect(fs.existsSync(result!.previewPath)).toBe(false)
})
it('falls back to dcraw_emu when extracted embedded JPEG is corrupt', async () => {
let createdJpegPath = ''
mockExtractBinaryTag.mockImplementationOnce(async (_tag, _src, dest) => {
createdJpegPath = dest
fs.writeFileSync(dest, 'corrupt-jpeg')
})
mockRead.mockResolvedValueOnce({ Orientation: 1 } as never)
mockMetadata.mockRejectedValueOnce(new Error('Not a valid image'))
// Fallback dcraw_emu
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
expect(result).not.toBeNull()
expect(fs.existsSync(createdJpegPath)).toBe(false) // Embedded jpeg was cleaned up
expect(fs.existsSync(result!.previewPath)).toBe(true)
result!.cleanup()
expect(fs.existsSync(result!.previewPath)).toBe(false)
})
it('returns null when both embedded extraction and dcraw_emu fail', async () => {
mockExtractBinaryTag
.mockRejectedValueOnce(new Error('fail'))
.mockRejectedValueOnce(new Error('fail'))
.mockRejectedValueOnce(new Error('fail'))
mockRead.mockRejectedValueOnce(new Error('fail'))
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(new Error('dcraw failed'))
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
expect(result).toBeNull()
})
it('returns null and cleans up when validation fails', async () => {
let createdPath = ''
mockExtractBinaryTag.mockImplementationOnce(async (_tag, _src, dest) => {
createdPath = dest
fs.writeFileSync(dest, 'bad-jpeg')
it('returns null and cleans up when decoded TIFF is corrupt', async () => {
mockExtractBinaryTag
.mockRejectedValueOnce(new Error('fail'))
.mockRejectedValueOnce(new Error('fail'))
.mockRejectedValueOnce(new Error('fail'))
let createdTiff = ''
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
createdTiff = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(createdTiff, 'corrupt-tiff')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
mockRead.mockResolvedValueOnce({ Orientation: 1 } as never)
mockMetadata.mockRejectedValueOnce(new Error('Not a valid image'))
mockMetadata.mockRejectedValueOnce(new Error('Invalid TIFF'))
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
expect(result).toBeNull()
expect(fs.existsSync(createdPath)).toBe(false)
expect(fs.existsSync(createdTiff)).toBe(false)
})
it('falls back to dcraw_emu when embedded preview extraction times out', async () => {
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
try {
process.env.EXIFTOOL_TIMEOUT_MS = '20'
// Hang on extractBinaryTag
mockExtractBinaryTag.mockImplementationOnce(
() => new Promise((resolve) => setTimeout(resolve, 300)),
)
mockRead.mockResolvedValueOnce({
ImageWidth: 6000,
ImageHeight: 4000,
Orientation: 1,
} as never)
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
fs.writeFileSync(tiffDest, 'tiff-data')
const callback = typeof _opts === 'function' ? _opts : cb
if (callback) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(callback as any)(null, { stdout: '', stderr: '' })
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return {} as any
})
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
expect(result).not.toBeNull()
expect(mockExecFile).toHaveBeenCalled()
expect(result!.width).toBe(6000)
expect(result!.height).toBe(4000)
result!.cleanup()
} finally {
if (origEnv !== undefined) {
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
} else {
delete process.env.EXIFTOOL_TIMEOUT_MS
}
}
})
})
describe('withTimeout', () => {
it('resolves when promise settles before timeout', async () => {
const result = await withTimeout(Promise.resolve('ok'), 1000)
expect(result).toBe('ok')
})
it('rejects with timeout error when promise exceeds timeout', async () => {
const hanging = new Promise((resolve) => setTimeout(resolve, 500))
await expect(withTimeout(hanging, 20, 'TestTask')).rejects.toThrow(
'TestTask timed out after 20ms',
)
})
it('propagates underlying promise rejection before timeout', async () => {
const failing = Promise.reject(new Error('fail fast'))
await expect(withTimeout(failing, 1000, 'TestTask')).rejects.toThrow('fail fast')
})
it('prevents unhandled rejection if promise rejects after timeout', async () => {
const delayedReject = new Promise((_, reject) =>
setTimeout(() => reject(new Error('late error')), 40),
)
await expect(withTimeout(delayedReject, 10, 'TestTask')).rejects.toThrow(
'TestTask timed out after 10ms',
)
await new Promise((resolve) => setTimeout(resolve, 60))
})
})
+265 -29
View File
@@ -1,15 +1,37 @@
import { execFile } from 'child_process'
import { exiftool } from 'exiftool-vendored'
import sharp from 'sharp'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import sharp from 'sharp'
import { promisify } from 'util'
import { logger } from '@shumai/core/src/logger'
const execFileAsync = promisify(execFile)
export interface RawExtractResult {
previewPath: string
cleanup: () => void
/** EXIF orientation from the RAW container (1–8), or undefined if absent. */
/** EXIF orientation from the RAW container (1–8), or undefined if absent or already applied. */
orientation?: number
/** Original unoriented or oriented image dimensions from EXIF if available */
rawWidth?: number
rawHeight?: number
}
export interface RawPreviewResult {
previewPath: string
cleanup: () => void
width: number
height: number
orientation?: number
rawWidth?: number
rawHeight?: number
}
export interface DecodeRawOptions {
timeoutMs?: number
signal?: AbortSignal
}
/**
@@ -30,6 +52,41 @@ export const EXIF_ORIENTATION_TO_ROTATION: Record<
8: { angle: 270 }, // Rotate 270° CW
}
const DEFAULT_EXIFTOOL_TIMEOUT_MS = 15_000
function getExifToolTimeoutMs(): number {
const envVal = process.env.EXIFTOOL_TIMEOUT_MS
if (envVal) {
const parsed = parseInt(envVal, 10)
if (!isNaN(parsed) && parsed > 0) {
return parsed
}
}
return DEFAULT_EXIFTOOL_TIMEOUT_MS
}
/**
* Wraps a promise with a timeout. If the timeout triggers first,
* the returned promise rejects with a timeout error and attaches a no-op handler
* to the underlying promise to prevent unhandled rejections.
*/
export function withTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
operationName = 'Operation',
): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined
const timeoutPromise = new Promise<never>((_, reject) => {
timer = setTimeout(() => {
reject(new Error(`${operationName} timed out after ${timeoutMs}ms`))
}, timeoutMs)
})
promise.catch(() => {})
return Promise.race([promise, timeoutPromise]).finally(() => {
if (timer) clearTimeout(timer)
})
}
/**
* Attempt to extract an embedded JPEG preview from a camera RAW file to a temporary file.
*
@@ -61,32 +118,58 @@ export async function extractEmbeddedJpeg(
}
}
const exiftoolTimeoutMs = getExifToolTimeoutMs()
for (const tag of tags) {
const candidatePath = path.join(workDir, `preview-${tag}-${Date.now()}.jpg`)
try {
await exiftool.extractBinaryTag(tag, rawFilePath, candidatePath)
await withTimeout(
exiftool.extractBinaryTag(tag, rawFilePath, candidatePath),
exiftoolTimeoutMs,
`exiftool.extractBinaryTag(${tag})`,
)
if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).size > 0) {
logger.debug({ tag, rawFilePath, candidatePath }, 'Extracted embedded JPEG from RAW')
// Read orientation from the RAW container's EXIF metadata.
// The extracted JPEG binary stream often does NOT carry its own
// orientation tag, so we must read it from the parent RAW file
// to avoid incorrectly-rotated previews.
// Read orientation and container dimensions from the RAW container's EXIF metadata.
let orientation: number | undefined
let rawWidth: number | undefined
let rawHeight: number | undefined
try {
const exif = await exiftool.read(rawFilePath)
const exif = await withTimeout(
exiftool.read(rawFilePath),
exiftoolTimeoutMs,
'exiftool.read',
)
const rawOrientation = exif.Orientation
if (typeof rawOrientation === 'number' && rawOrientation >= 1 && rawOrientation <= 8) {
orientation = rawOrientation
}
const w =
typeof exif.ImageWidth === 'number'
? exif.ImageWidth
: typeof exif.ExifImageWidth === 'number'
? exif.ExifImageWidth
: undefined
const h =
typeof exif.ImageHeight === 'number'
? exif.ImageHeight
: typeof exif.ExifImageHeight === 'number'
? exif.ExifImageHeight
: undefined
if (w && h && w > 0 && h > 0) {
rawWidth = w
rawHeight = h
}
} catch {
logger.debug({ rawFilePath }, 'Could not read EXIF orientation from RAW container')
}
return { previewPath: candidatePath, cleanup, orientation }
return { previewPath: candidatePath, cleanup, orientation, rawWidth, rawHeight }
}
} catch {
logger.debug({ tag, rawFilePath }, `Could not extract ${tag} from RAW file`)
} catch (err) {
const isTimeout = err instanceof Error && err.message.includes('timed out')
logger.debug({ tag, rawFilePath, err, isTimeout }, `Could not extract ${tag} from RAW file`)
try {
if (fs.existsSync(candidatePath)) {
fs.rmSync(candidatePath, { force: true })
@@ -94,6 +177,13 @@ export async function extractEmbeddedJpeg(
} catch {
// ignore
}
if (isTimeout) {
logger.warn(
{ rawFilePath, tag, timeoutMs: exiftoolTimeoutMs },
'ExifTool extraction timed out; skipping remaining tags to avoid further delays',
)
break
}
}
}
@@ -103,10 +193,124 @@ export async function extractEmbeddedJpeg(
}
/**
* Validate that an extracted JPEG file is decodable.
* Fallback RAW decoder using dcraw_emu (from LibRaw).
*
* Used when no usable embedded JPEG preview exists in the RAW file.
*
* Decoding options:
* - Camera white balance and sRGB output (-w -o 1)
* - Output temporary TIFF (-T -Z <tempPath>)
* - Half-size processing (-h) enabled only when longest dimension > 8192px
*
* LibRaw auto-rotates the output image to upright orientation based on camera EXIF,
* so the returned orientation is undefined to avoid double-rotation in downstream Sharp.
*/
export async function decodeRawWithDcraw(
rawFilePath: string,
targetDir?: string,
options?: DecodeRawOptions,
): Promise<RawExtractResult | null> {
const ownTempDir = !targetDir
const workDir = targetDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'raw-decode-'))
const tempTiffPath = path.join(
workDir,
`decoded-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.tiff`,
)
const cleanup = () => {
try {
if (ownTempDir && fs.existsSync(workDir)) {
fs.rmSync(workDir, { recursive: true, force: true })
} else if (fs.existsSync(tempTiffPath)) {
fs.rmSync(tempTiffPath, { force: true })
}
} catch (err) {
logger.warn({ workDir, tempTiffPath, err }, 'Failed to clean up dcraw temporary files')
}
}
// 1. Probe RAW dimensions and orientation via ExifTool to check for 8192px threshold
let useHalfSize = false
let rawWidth: number | undefined
let rawHeight: number | undefined
try {
const exiftoolTimeoutMs = getExifToolTimeoutMs()
const exif = await withTimeout(exiftool.read(rawFilePath), exiftoolTimeoutMs, 'exiftool.read')
const w =
typeof exif.ImageWidth === 'number'
? exif.ImageWidth
: typeof exif.ExifImageWidth === 'number'
? exif.ExifImageWidth
: undefined
const h =
typeof exif.ImageHeight === 'number'
? exif.ImageHeight
: typeof exif.ExifImageHeight === 'number'
? exif.ExifImageHeight
: undefined
if (w && h && w > 0 && h > 0) {
const longest = Math.max(w, h)
if (longest > 8192) {
useHalfSize = true
}
// Compute upright raw dimensions matching dcraw_emu's auto-rotation
const rawOrientation = exif.Orientation
const isSwapped =
typeof rawOrientation === 'number' && rawOrientation >= 5 && rawOrientation <= 8
rawWidth = isSwapped ? h : w
rawHeight = isSwapped ? w : h
}
} catch (err) {
logger.debug({ rawFilePath, err }, 'Could not read EXIF metadata for RAW dimension check')
}
// 2. Build dcraw_emu arguments
const args = ['-w', '-o', '1']
if (useHalfSize) {
args.push('-h')
}
args.push('-T', '-Z', tempTiffPath, rawFilePath)
const dcrawBin = process.env.DCRAW_EMU_PATH || 'dcraw_emu'
const envTimeout = process.env.DCRAW_EMU_TIMEOUT_MS
? parseInt(process.env.DCRAW_EMU_TIMEOUT_MS, 10)
: undefined
const timeoutMs = options?.timeoutMs ?? (envTimeout && !isNaN(envTimeout) ? envTimeout : 60_000)
try {
logger.debug({ dcrawBin, args, rawFilePath }, 'Executing dcraw_emu fallback')
await execFileAsync(dcrawBin, args, {
timeout: timeoutMs,
signal: options?.signal,
})
if (!fs.existsSync(tempTiffPath) || fs.statSync(tempTiffPath).size === 0) {
logger.warn({ rawFilePath, tempTiffPath }, 'dcraw_emu did not produce a valid output TIFF')
cleanup()
return null
}
return {
previewPath: tempTiffPath,
cleanup,
orientation: undefined, // dcraw_emu already auto-rotates upright based on camera EXIF
rawWidth,
rawHeight,
}
} catch (err) {
logger.warn({ rawFilePath, err }, 'dcraw_emu failed to decode RAW file')
cleanup()
return null
}
}
/**
* Validate that an extracted or decoded image file (JPEG, TIFF, etc.) is decodable.
* Returns dimensions if valid, null if corrupt/unusable.
*/
export async function validateExtractedJpeg(
export async function validateExtractedImage(
filePath: string,
): Promise<{ width: number; height: number } | null> {
try {
@@ -120,34 +324,66 @@ export async function validateExtractedJpeg(
}
}
/** Backward compatibility alias */
export const validateExtractedJpeg = validateExtractedImage
/**
* Extract and validate an embedded JPEG from a RAW file to a temporary file.
* Extract and validate a preview from a RAW file to a temporary file.
*
* 1. Attempts ExifTool embedded JPEG extraction first.
* 2. If no embedded preview is available or extraction fails, falls back to dcraw_emu decoding.
*
* Returns a validated file path + dimensions + orientation + cleanup function, or null.
*/
export async function extractAndValidateRawPreview(
rawFilePath: string,
targetDir?: string,
): Promise<{
previewPath: string
cleanup: () => void
width: number
height: number
orientation?: number
} | null> {
): Promise<RawPreviewResult | null> {
// 1. Try ExifTool embedded JPEG extraction first
const extracted = await extractEmbeddedJpeg(rawFilePath, targetDir)
if (!extracted) return null
const dims = await validateExtractedJpeg(extracted.previewPath)
if (!dims) {
logger.warn({ rawFilePath }, 'Extracted JPEG from RAW is not decodable, skipping preview')
if (extracted) {
const dims = await validateExtractedImage(extracted.previewPath)
if (dims) {
return {
previewPath: extracted.previewPath,
cleanup: extracted.cleanup,
orientation: extracted.orientation,
rawWidth: extracted.rawWidth,
rawHeight: extracted.rawHeight,
...dims,
}
}
logger.warn(
{ rawFilePath },
'Extracted JPEG from RAW is not decodable, falling back to dcraw_emu',
)
extracted.cleanup()
} else {
logger.info(
{ rawFilePath },
'No usable embedded JPEG found in RAW file, falling back to dcraw_emu',
)
}
// 2. Fallback to dcraw_emu
const decoded = await decodeRawWithDcraw(rawFilePath, targetDir)
if (!decoded) {
return null
}
const dims = await validateExtractedImage(decoded.previewPath)
if (!dims) {
logger.warn({ rawFilePath }, 'Decoded TIFF from dcraw_emu is not decodable, skipping preview')
decoded.cleanup()
return null
}
return {
previewPath: extracted.previewPath,
cleanup: extracted.cleanup,
orientation: extracted.orientation,
previewPath: decoded.previewPath,
cleanup: decoded.cleanup,
orientation: decoded.orientation,
rawWidth: decoded.rawWidth,
rawHeight: decoded.rawHeight,
...dims,
}
}
+793 -1
View File
@@ -7,7 +7,11 @@ import {
getDefaultBitrateBps,
calculateMaxBitrate,
H264_ENCODER_CONFIGS,
isHwDecodeEnabled,
getHwDecodeStallTimeoutMs,
HW_DECODE_CONFIGS,
buildSdrToneMapFilterChain,
getDriDevice,
getVaapiDevice,
parseBitrateKbps,
buildHlsMasterPlaylist,
@@ -506,6 +510,26 @@ describe('TranscodeService', () => {
expect(info.mimeType).toBe('')
})
it('should prioritize rawWidth and rawHeight in getImageInfo when available', async () => {
const rawPath = path.join(tempDir, 'highres.arw')
const cleanupSpy = vi.fn()
vi.spyOn(rawExtract, 'extractAndValidateRawPreview').mockResolvedValueOnce({
previewPath: '/tmp/decoded-half.tiff',
cleanup: cleanupSpy,
width: 4752,
height: 3168,
rawWidth: 9504,
rawHeight: 6336,
orientation: undefined,
})
const info = await transcodeService.getImageInfo(rawPath)
expect(info.originalWidth).toBe(9504)
expect(info.originalHeight).toBe(6336)
expect(info.mimeType).toBe('jpeg')
expect(cleanupSpy).toHaveBeenCalled()
})
it('should transcode RAW image by extracting preview to temp file and feeding to sharp', async () => {
const rawPath = path.join(tempDir, 'sample.nef')
const outputFile = path.join(tempDir, 'output-raw.webp')
@@ -528,6 +552,31 @@ describe('TranscodeService', () => {
expect(cleanupSpy).toHaveBeenCalled()
})
it('should transcode RAW image from dcraw_emu fallback TIFF without double-rotating', async () => {
const rawPath = path.join(tempDir, 'fallback.dng')
const outputFile = path.join(tempDir, 'output-fallback.webp')
const cleanupSpy = vi.fn()
vi.spyOn(rawExtract, 'extractAndValidateRawPreview').mockResolvedValueOnce({
previewPath: '/tmp/decoded-dcraw.tiff',
cleanup: cleanupSpy,
width: 3168,
height: 4752,
rawWidth: 6336,
rawHeight: 9504,
orientation: undefined,
})
await transcodeService.transcodeImage(rawPath, outputFile, 1920, 85)
expect(sharp).toHaveBeenCalledWith('/tmp/decoded-dcraw.tiff', { limitInputPixels: false })
const mockSharp = vi.mocked(sharp).mock.results[vi.mocked(sharp).mock.results.length - 1].value
expect(mockSharp.rotate).not.toHaveBeenCalled()
expect(mockSharp.toColorspace).toHaveBeenCalledWith('srgb')
expect(mockSharp.webp).toHaveBeenCalledWith({ quality: 85 })
expect(mockSharp.toFile).toHaveBeenCalledWith(outputFile)
expect(cleanupSpy).toHaveBeenCalled()
})
it('should apply orientation rotation when transcoding RAW image', async () => {
const rawPath = path.join(tempDir, 'rotated.cr3')
const outputFile = path.join(tempDir, 'output-rotated.webp')
@@ -1325,12 +1374,20 @@ describe('TranscodeService', () => {
})
describe('Hardware Acceleration & Encoder Resolution', () => {
let origHwDecode: string | undefined
beforeEach(() => {
transcodeService.clearEncodersCache()
// These tests cover the software-decode VAAPI path; the GPU decode path
// (on by default) is covered in 'Hardware decode' below.
origHwDecode = process.env.SHUMAI_HW_DECODE
process.env.SHUMAI_HW_DECODE = 'false'
})
afterEach(() => {
vi.restoreAllMocks()
if (origHwDecode === undefined) delete process.env.SHUMAI_HW_DECODE
else process.env.SHUMAI_HW_DECODE = origHwDecode
})
it('should return platform candidates correctly', () => {
@@ -1345,21 +1402,31 @@ describe('TranscodeService', () => {
'h264_nvenc',
'h264_vaapi',
'h264_qsv',
'h264_rkmpp',
'h264_amf',
])
})
it('getVaapiDevice should return device from SHUMAI_VAAPI_DEVICE or VAAPI_DEVICE env var', () => {
it('getDriDevice should return device from SHUMAI_HW_DEVICE, SHUMAI_VAAPI_DEVICE, or VAAPI_DEVICE env var', () => {
const origHw = process.env.SHUMAI_HW_DEVICE
const origShumai = process.env.SHUMAI_VAAPI_DEVICE
const origVaapi = process.env.VAAPI_DEVICE
try {
process.env.SHUMAI_HW_DEVICE = '/dev/dri/custom0'
expect(getDriDevice()).toBe('/dev/dri/custom0')
delete process.env.SHUMAI_HW_DEVICE
process.env.SHUMAI_VAAPI_DEVICE = '/dev/dri/custom1'
expect(getDriDevice()).toBe('/dev/dri/custom1')
expect(getVaapiDevice()).toBe('/dev/dri/custom1')
delete process.env.SHUMAI_VAAPI_DEVICE
process.env.VAAPI_DEVICE = '/dev/dri/custom2'
expect(getDriDevice()).toBe('/dev/dri/custom2')
expect(getVaapiDevice()).toBe('/dev/dri/custom2')
} finally {
if (origHw) process.env.SHUMAI_HW_DEVICE = origHw
else delete process.env.SHUMAI_HW_DEVICE
if (origShumai) process.env.SHUMAI_VAAPI_DEVICE = origShumai
else delete process.env.SHUMAI_VAAPI_DEVICE
if (origVaapi) process.env.VAAPI_DEVICE = origVaapi
@@ -2531,6 +2598,731 @@ describe('TranscodeService', () => {
})
})
describe('Hardware decode', () => {
let origHwDecode: string | undefined
let origStall: string | undefined
type FfmpegCallback = (err: Error | null, result: { stdout: string; stderr: string }) => void
// execFile is called as (cmd, args, cb) or (cmd, args, options, cb); the
// GPU-decode attempt always passes options (signal + SIGKILL watchdog).
const callbackOf = (optionsOrCallback: unknown, maybeCallback?: unknown) =>
(typeof optionsOrCallback === 'function'
? optionsOrCallback
: maybeCallback) as FfmpegCallback
const mockFfmpeg = (onCall: (n: number, cb: FfmpegCallback, options: unknown) => void) =>
vi
.mocked(execFile)
.mockImplementation(
(
_cmd: unknown,
_args: unknown,
optionsOrCallback: unknown,
maybeCallback?: unknown,
): ReturnType<typeof child_process.execFile> => {
const n = vi.mocked(execFile).mock.calls.length
onCall(n, callbackOf(optionsOrCallback, maybeCallback), optionsOrCallback)
return {} as ReturnType<typeof child_process.execFile>
},
)
const failOn =
(...failing: number[]) =>
(n: number, cb: FfmpegCallback) =>
failing.includes(n)
? cb(new Error(`ffmpeg failed on call ${n}`), { stdout: '', stderr: '' })
: cb(null, { stdout: '', stderr: '' })
const calls = () => vi.mocked(child_process.execFile).mock.calls.map((c) => c[1] as string[])
const filterOf = (args: string[]) => args[args.indexOf('-filter_complex') + 1]
const vaapiScale = (w: number, h: number) => HW_DECODE_CONFIGS.h264_vaapi!.scaleFilter(w, h)
const nvencScale = (w: number, h: number) => HW_DECODE_CONFIGS.h264_nvenc!.scaleFilter(w, h)
const qsvScale = (w: number, h: number) => HW_DECODE_CONFIGS.h264_qsv!.scaleFilter(w, h)
const rkmppScale = (w: number, h: number) => HW_DECODE_CONFIGS.h264_rkmpp!.scaleFilter(w, h)
beforeEach(() => {
transcodeService.clearEncodersCache()
origHwDecode = process.env.SHUMAI_HW_DECODE
origStall = process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT
delete process.env.SHUMAI_HW_DECODE
delete process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT
vi.spyOn(transcodeService, 'selectH264Encoder').mockResolvedValue(
H264_ENCODER_CONFIGS.h264_vaapi,
)
vi.spyOn(transcodeService, 'getVaapiDevice').mockReturnValue('/dev/dri/renderD128')
vi.spyOn(transcodeService, 'getDriDevice').mockReturnValue('/dev/dri/renderD128')
vi.spyOn(transcodeService, 'getVideoRotation').mockResolvedValue(0)
})
afterEach(() => {
vi.restoreAllMocks()
if (origHwDecode === undefined) delete process.env.SHUMAI_HW_DECODE
else process.env.SHUMAI_HW_DECODE = origHwDecode
if (origStall === undefined) delete process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT
else process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT = origStall
})
it('isHwDecodeEnabled should default to true and honor false/0/off', () => {
expect(isHwDecodeEnabled()).toBe(true)
for (const value of ['false', '0', 'off', ' FALSE ']) {
process.env.SHUMAI_HW_DECODE = value
expect(isHwDecodeEnabled()).toBe(false)
}
process.env.SHUMAI_HW_DECODE = 'true'
expect(isHwDecodeEnabled()).toBe(true)
})
it('getHwDecodeStallTimeoutMs should default to 120s and honor SHUMAI_HW_DECODE_STALL_TIMEOUT', () => {
expect(getHwDecodeStallTimeoutMs()).toBe(120_000)
process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT = '30'
expect(getHwDecodeStallTimeoutMs()).toBe(30_000)
process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT = 'nope'
expect(getHwDecodeStallTimeoutMs()).toBe(120_000)
})
it('should have hardware decode configs for nvenc, qsv, vaapi, and rkmpp', () => {
expect(Object.keys(HW_DECODE_CONFIGS)).toEqual([
'h264_nvenc',
'h264_qsv',
'h264_vaapi',
'h264_rkmpp',
])
expect(nvencScale(1920, 1080)).toBe(
'scale_cuda=w=1920:h=1080:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12',
)
expect(qsvScale(1920, 1080)).toBe('scale_qsv=w=1920:h=1080:async_depth=4:mode=hq:format=nv12')
expect(vaapiScale(1920, 1080)).toBe(
'scale_vaapi=w=1920:h=1080:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12:mode=hq',
)
expect(rkmppScale(1920, 1080)).toBe(
'scale_rkrga=w=1920:h=1080:format=nv12:afbc=1:async_depth=4',
)
expect(HW_DECODE_CONFIGS.h264_qsv!.getInputArgs!('/dev/dri/renderD128')).toEqual([
'-hwaccel',
'qsv',
'-hwaccel_output_format',
'qsv',
'-async_depth',
'4',
'-threads',
'1',
'-qsv_device',
'/dev/dri/renderD128',
])
expect(HW_DECODE_CONFIGS.h264_qsv!.getInputArgs!()).toEqual([
'-hwaccel',
'qsv',
'-hwaccel_output_format',
'qsv',
'-async_depth',
'4',
'-threads',
'1',
])
})
it('transcodeVideo should decode and scale on the GPU by default with vaapi', async () => {
mockFfmpeg(failOn())
const loggerSpy = vi.spyOn(logger, 'info')
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec.mp4'),
width: 1280,
height: 720,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceVideoBitrate: 600_000,
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
const args = calls()[0]
expect(args).toEqual(
expect.arrayContaining([
'-init_hw_device',
'vaapi=accel:/dev/dri/renderD128',
'-hwaccel',
'vaapi',
'-hwaccel_device',
'accel',
'-hwaccel_output_format',
'vaapi',
'-noautoscale',
'-c:v',
'h264_vaapi',
'-rc_mode',
'3',
]),
)
// hwaccel options must precede the input; -noautoscale is an output option
expect(args.indexOf('-hwaccel')).toBeLessThan(args.indexOf('-i'))
expect(args.indexOf('-noautoscale')).toBeGreaterThan(args.indexOf('-filter_complex'))
expect(filterOf(args)).toBe(`[0:V]${vaapiScale(1280, 720)},fps=24[vout]`)
expect(filterOf(args)).not.toContain('hwupload')
expect(args).not.toContain('-pix_fmt')
// the GPU attempt runs under the stall watchdog
expect(vi.mocked(child_process.execFile).mock.calls[0][2]).toEqual(
expect.objectContaining({ killSignal: 'SIGKILL', signal: expect.any(AbortSignal) }),
)
expect(loggerSpy).toHaveBeenCalledWith(
expect.objectContaining({ encoder: 'h264_vaapi', hwDecode: true }),
'Starting video transcoding',
)
})
it('transcodeVideo should decode and scale on the GPU by default with nvenc', async () => {
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(
H264_ENCODER_CONFIGS.h264_nvenc,
)
mockFfmpeg(failOn())
const loggerSpy = vi.spyOn(logger, 'info')
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_nvenc.mp4'),
width: 1280,
height: 720,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceVideoBitrate: 600_000,
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
const args = calls()[0]
expect(args).toEqual(
expect.arrayContaining([
'-hwaccel',
'cuda',
'-hwaccel_output_format',
'cuda',
'-threads',
'1',
'-noautoscale',
'-c:v',
'h264_nvenc',
]),
)
expect(args.indexOf('-hwaccel')).toBeLessThan(args.indexOf('-i'))
expect(args.indexOf('-noautoscale')).toBeGreaterThan(args.indexOf('-filter_complex'))
expect(filterOf(args)).toBe(`[0:V]${nvencScale(1280, 720)},fps=24[vout]`)
expect(filterOf(args)).not.toContain('hwupload')
expect(args).not.toContain('-pix_fmt')
expect(loggerSpy).toHaveBeenCalledWith(
expect.objectContaining({ encoder: 'h264_nvenc', hwDecode: true }),
'Starting video transcoding',
)
})
it('transcodeVideo should decode and scale on the GPU by default with qsv', async () => {
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(H264_ENCODER_CONFIGS.h264_qsv)
mockFfmpeg(failOn())
const loggerSpy = vi.spyOn(logger, 'info')
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_qsv.mp4'),
width: 1280,
height: 720,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceVideoBitrate: 600_000,
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
const args = calls()[0]
expect(args).toEqual(
expect.arrayContaining([
'-hwaccel',
'qsv',
'-hwaccel_output_format',
'qsv',
'-async_depth',
'4',
'-threads',
'1',
'-noautoscale',
'-c:v',
'h264_qsv',
]),
)
expect(args).toContain('-qsv_device')
expect(args).toContain('/dev/dri/renderD128')
expect(args.indexOf('-hwaccel')).toBeLessThan(args.indexOf('-i'))
expect(args.indexOf('-noautoscale')).toBeGreaterThan(args.indexOf('-filter_complex'))
expect(filterOf(args)).toBe(`[0:V]${qsvScale(1280, 720)},fps=24[vout]`)
expect(filterOf(args)).not.toContain('hwupload')
expect(args).not.toContain('-pix_fmt')
expect(loggerSpy).toHaveBeenCalledWith(
expect.objectContaining({ encoder: 'h264_qsv', hwDecode: true }),
'Starting video transcoding',
)
})
it('transcodeVideo should decode and scale on the GPU by default with rkmpp', async () => {
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(
H264_ENCODER_CONFIGS.h264_rkmpp,
)
mockFfmpeg(failOn())
const loggerSpy = vi.spyOn(logger, 'info')
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_rkmpp.mp4'),
width: 1280,
height: 720,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceVideoBitrate: 600_000,
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
const args = calls()[0]
expect(args).toEqual(
expect.arrayContaining([
'-hwaccel',
'rkmpp',
'-hwaccel_output_format',
'drm_prime',
'-afbc',
'rga',
'-noautoscale',
'-c:v',
'h264_rkmpp',
]),
)
expect(args.indexOf('-hwaccel')).toBeLessThan(args.indexOf('-i'))
expect(args.indexOf('-noautoscale')).toBeGreaterThan(args.indexOf('-filter_complex'))
expect(filterOf(args)).toBe(`[0:V]${rkmppScale(1280, 720)},fps=24[vout]`)
expect(filterOf(args)).not.toContain('hwupload')
expect(args).not.toContain('-pix_fmt')
expect(loggerSpy).toHaveBeenCalledWith(
expect.objectContaining({ encoder: 'h264_rkmpp', hwDecode: true }),
'Starting video transcoding',
)
})
it('transcodeVideo should fall back to software decode when qsv has no DRI device on Linux', async () => {
const origPlatform = process.platform
try {
Object.defineProperty(process, 'platform', { value: 'linux', configurable: true })
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(
H264_ENCODER_CONFIGS.h264_qsv,
)
vi.spyOn(transcodeService, 'getDriDevice').mockReturnValue(null)
vi.spyOn(transcodeService, 'getVaapiDevice').mockReturnValue(null)
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_qsv_no_dri.mp4'),
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()).toHaveLength(1)
expect(calls()[0]).not.toContain('-hwaccel')
expect(calls()[0]).toContain('h264_qsv')
} finally {
Object.defineProperty(process, 'platform', { value: origPlatform, configurable: true })
}
})
it('transcodeVideo should use the provided sourceRotation instead of probing', async () => {
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'portrait.mov',
outputFile: path.join(tempDir, 'out_hwdec_rotation_param.mp4'),
width: 1080,
height: 1920,
hardwareAcceleration: 'auto',
sourceRotation: -90,
})
expect(transcodeService.getVideoRotation).not.toHaveBeenCalled()
expect(calls()[0]).not.toContain('-hwaccel')
expect(filterOf(calls()[0])).toContain('hwupload')
})
it('transcodeVideo should probe rotation when sourceRotation is not provided', async () => {
vi.mocked(transcodeService.getVideoRotation).mockResolvedValue(90)
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'portrait.mov',
outputFile: path.join(tempDir, 'out_hwdec_rotated.mp4'),
width: 1080,
height: 1920,
hardwareAcceleration: 'auto',
})
expect(transcodeService.getVideoRotation).toHaveBeenCalledWith('portrait.mov', undefined)
expect(calls()).toHaveLength(1)
expect(calls()[0]).not.toContain('-hwaccel')
expect(filterOf(calls()[0])).toContain('hwupload')
})
it('transcodeVideo should retry with software decode + hardware encode when GPU decode fails', async () => {
const outputFile = path.join(tempDir, 'out_hwdec_retry.mp4')
mockFfmpeg(failOn(1))
const warnSpy = vi.spyOn(logger, 'warn')
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile,
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()).toHaveLength(2)
expect(calls()[0]).toContain('-hwaccel')
expect(calls()[1]).not.toContain('-hwaccel')
expect(calls()[1]).not.toContain('-noautoscale')
expect(calls()[1]).toContain('h264_vaapi')
expect(filterOf(calls()[1])).toContain('format=nv12,hwupload=extra_hw_frames=64')
expect(warnSpy).toHaveBeenCalledWith(
expect.objectContaining({ encoder: 'h264_vaapi', outputFile }),
'Hardware decode failed; retrying with software decode and hardware encode',
)
})
it('transcodeVideo should kill a stalled GPU decode attempt and fall back', async () => {
process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT = '0.05'
const outputFile = path.join(tempDir, 'out_hwdec_stall.mp4')
const warnSpy = vi.spyOn(logger, 'warn')
let killSignal: unknown
mockFfmpeg((n, cb, options) => {
if (n === 1) {
// simulate a hung ffmpeg: never exits on its own, only when killed
const opts = options as { signal: AbortSignal; killSignal: string }
killSignal = opts.killSignal
opts.signal.addEventListener('abort', () => {
const err = new Error('The operation was aborted')
err.name = 'AbortError'
cb(err, { stdout: '', stderr: '' })
})
} else {
cb(null, { stdout: '', stderr: '' })
}
})
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile,
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(killSignal).toBe('SIGKILL')
expect(calls()).toHaveLength(2)
expect(calls()[1]).not.toContain('-hwaccel')
expect(warnSpy).toHaveBeenCalledWith(
expect.objectContaining({
err: expect.objectContaining({ message: expect.stringContaining('made no progress') }),
}),
'Hardware decode failed; retrying with software decode and hardware encode',
)
})
it('transcodeVideo should fall back to libx264 when GPU decode and vaapi encode both fail', async () => {
const outputFile = path.join(tempDir, 'out_hwdec_x264.mp4')
let existedBeforeRetry = true
mockFfmpeg((n, cb) => {
if (n === 1) {
fs.writeFileSync(outputFile, 'partial video data')
cb(new Error('hw decode failed'), { stdout: '', stderr: '' })
} else if (n === 2) {
existedBeforeRetry = fs.existsSync(outputFile)
cb(new Error('vaapi encode failed'), { stdout: '', stderr: '' })
} else {
cb(null, { stdout: '', stderr: '' })
}
})
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile,
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()).toHaveLength(3)
expect(existedBeforeRetry).toBe(false)
expect(calls()[0]).toContain('-hwaccel')
expect(calls()[1]).toContain('h264_vaapi')
expect(calls()[1]).not.toContain('-hwaccel')
expect(calls()[2]).toContain('libx264')
})
it('transcodeVideo should not retry when the GPU decode attempt was aborted by the caller', async () => {
const abortController = new AbortController()
mockFfmpeg((_n, cb) => {
abortController.abort()
const abortErr = new Error('The operation was aborted')
abortErr.name = 'AbortError'
cb(abortErr, { stdout: '', stderr: '' })
})
await expect(
transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_abort.mp4'),
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
signal: abortController.signal,
}),
).rejects.toThrow('aborted')
expect(vi.mocked(child_process.execFile)).toHaveBeenCalledTimes(1)
})
it('transcodeVideo should keep software decode for watermark overlays', async () => {
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_overlay.mp4'),
width: 1280,
height: 720,
overlayFile: 'overlay.png',
hardwareAcceleration: 'auto',
})
expect(calls()).toHaveLength(1)
expect(calls()[0]).not.toContain('-hwaccel')
expect(filterOf(calls()[0])).toContain('overlay=0:0')
expect(filterOf(calls()[0])).toContain('hwupload')
})
it('transcodeVideo should keep software decode for HDR sources', async () => {
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'input.mov',
outputFile: path.join(tempDir, 'out_hwdec_hdr.mp4'),
width: 1920,
height: 1080,
sourceHdrType: 'pq',
hardwareAcceleration: 'auto',
})
expect(calls()[0]).not.toContain('-hwaccel')
expect(filterOf(calls()[0])).toContain('hwupload')
})
it('transcodeVideo should keep software decode when SHUMAI_HW_DECODE=false', async () => {
process.env.SHUMAI_HW_DECODE = 'false'
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, 'out_hwdec_disabled.mp4'),
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()[0]).not.toContain('-hwaccel')
expect(filterOf(calls()[0])).toContain('hwupload')
})
it('transcodeVideo should not use GPU decode for encoders without a hardware decode config', async () => {
for (const encoder of ['libx264', 'h264_amf', 'h264_videotoolbox'] as const) {
vi.mocked(child_process.execFile).mockClear()
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(
H264_ENCODER_CONFIGS[encoder],
)
mockFfmpeg(failOn())
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: path.join(tempDir, `out_hwdec_${encoder}.mp4`),
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()).toHaveLength(1)
expect(calls()[0]).not.toContain('-hwaccel')
expect(calls()[0]).toContain(encoder)
}
})
it('transcodeHlsRendition should decode and scale on the GPU by default with vaapi', async () => {
mockFfmpeg(failOn())
const loggerSpy = vi.spyOn(logger, 'info')
await transcodeService.transcodeHlsRendition({
inputFile: 'input.mp4',
outputDir: path.join(tempDir, 'hls_hwdec'),
width: 1920,
height: 1080,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
const args = calls()[0]
expect(args).toEqual(
expect.arrayContaining([
'-hwaccel',
'vaapi',
'-hwaccel_output_format',
'vaapi',
'-noautoscale',
'-f',
'hls',
]),
)
expect(args.indexOf('-hwaccel')).toBeLessThan(args.indexOf('-i'))
expect(filterOf(args)).toBe(`[0:V]${vaapiScale(1920, 1080)},fps=24[vout]`)
expect(transcodeService.getVideoRotation).not.toHaveBeenCalled()
expect(loggerSpy).toHaveBeenCalledWith(
expect.objectContaining({ encoder: 'h264_vaapi', hwDecode: true }),
'Starting HLS rendition transcoding',
)
})
it('transcodeHlsRendition should decode and scale on the GPU by default with nvenc and qsv', async () => {
// NVENC
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(
H264_ENCODER_CONFIGS.h264_nvenc,
)
mockFfmpeg(failOn())
await transcodeService.transcodeHlsRendition({
inputFile: 'input.mp4',
outputDir: path.join(tempDir, 'hls_hwdec_nvenc'),
width: 1920,
height: 1080,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
expect(calls()[0]).toContain('-hwaccel')
expect(calls()[0]).toContain('cuda')
expect(calls()[0]).toContain('-forced-idr')
expect(calls()[0]).toContain('1')
expect(filterOf(calls()[0])).toBe(`[0:V]${nvencScale(1920, 1080)},fps=24[vout]`)
// QSV
vi.mocked(child_process.execFile).mockClear()
vi.mocked(transcodeService.selectH264Encoder).mockResolvedValue(H264_ENCODER_CONFIGS.h264_qsv)
mockFfmpeg(failOn())
await transcodeService.transcodeHlsRendition({
inputFile: 'input.mp4',
outputDir: path.join(tempDir, 'hls_hwdec_qsv'),
width: 1920,
height: 1080,
frameRate: 24,
hardwareAcceleration: 'auto',
sourceRotation: 0,
})
expect(calls()).toHaveLength(1)
expect(calls()[0]).toContain('-hwaccel')
expect(calls()[0]).toContain('qsv')
expect(calls()[0]).toContain('-idr_interval')
expect(calls()[0]).toContain('0')
expect(filterOf(calls()[0])).toBe(`[0:V]${qsvScale(1920, 1080)},fps=24[vout]`)
})
it('transcodeHlsRendition should keep software decode when rotation cannot be determined', async () => {
vi.mocked(transcodeService.getVideoRotation).mockResolvedValue(null)
mockFfmpeg(failOn())
await transcodeService.transcodeHlsRendition({
inputFile: 'input.mp4',
outputDir: path.join(tempDir, 'hls_hwdec_unknown_rotation'),
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()[0]).not.toContain('-hwaccel')
})
it('transcodeHlsRendition should retry with software decode, then libx264, cleaning the output dir', async () => {
const outputDir = path.join(tempDir, 'hls_hwdec_fallback')
let dirExistedBeforeRetry = true
mockFfmpeg((n, cb) => {
if (n === 1) {
fs.writeFileSync(path.join(outputDir, 'segment_000.m4s'), 'partial')
cb(new Error('hw decode failed'), { stdout: '', stderr: '' })
} else if (n === 2) {
dirExistedBeforeRetry = fs.existsSync(path.join(outputDir, 'segment_000.m4s'))
cb(new Error('vaapi encode failed'), { stdout: '', stderr: '' })
} else {
cb(null, { stdout: '', stderr: '' })
}
})
await transcodeService.transcodeHlsRendition({
inputFile: 'input.mp4',
outputDir,
width: 1280,
height: 720,
hardwareAcceleration: 'auto',
})
expect(calls()).toHaveLength(3)
expect(dirExistedBeforeRetry).toBe(false)
expect(calls()[0]).toContain('-hwaccel')
expect(calls()[1]).not.toContain('-hwaccel')
expect(filterOf(calls()[1])).toContain('hwupload')
expect(calls()[2]).toContain('libx264')
})
it('getVideoRotation should read the display matrix, the rotate tag, and normalize', async () => {
vi.mocked(transcodeService.getVideoRotation).mockRestore()
const probe = (json: unknown) =>
vi
.mocked(execFile)
.mockImplementationOnce(
(
_cmd: unknown,
_args: unknown,
callback: unknown,
): ReturnType<typeof child_process.execFile> => {
;(callback as FfmpegCallback)(null, { stdout: JSON.stringify(json), stderr: '' })
return {} as ReturnType<typeof child_process.execFile>
},
)
/* eslint-disable @typescript-eslint/naming-convention */
probe({
streams: [{ side_data_list: [{ side_data_type: 'Display Matrix', rotation: -90 }] }],
})
/* eslint-enable @typescript-eslint/naming-convention */
expect(await transcodeService.getVideoRotation('a.mov')).toBe(270)
probe({ streams: [{ tags: { rotate: '180' } }] })
expect(await transcodeService.getVideoRotation('b.mp4')).toBe(180)
probe({ streams: [{}] })
expect(await transcodeService.getVideoRotation('c.mp4', 2)).toBe(0)
expect(vi.mocked(child_process.execFile).mock.calls.at(-1)?.[1]).toEqual(
expect.arrayContaining(['-select_streams', '2']),
)
probe({ streams: [] })
expect(await transcodeService.getVideoRotation('d.mp4')).toBeNull()
})
})
describe('HDR detection, tone mapping, and transcoding', () => {
it('buildSdrToneMapFilterChain constructs zscale + tonemap filter chain for PQ and HLG', () => {
const available = new Set(['zscale', 'tonemap'])
+475 -98
View File
@@ -194,6 +194,8 @@ export interface TranscodeVideoParams {
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
/** Source rotation in degrees from MediaMetadata; probed when omitted. */
sourceRotation?: number
streamIndex?: number
audioStreamIndex?: number
}
@@ -218,6 +220,8 @@ export interface TranscodeHlsRenditionParams {
sourceColorPrimaries?: string
sourceColorSpace?: string
segmentDuration?: number
/** Source rotation in degrees from MediaMetadata; probed when omitted. */
sourceRotation?: number
streamIndex?: number
audioStreamIndex?: number
}
@@ -240,6 +244,10 @@ export const H264_ENCODER_CONFIGS: Record<string, EncoderConfig> = {
name: 'h264_vaapi',
presetArgs: ['-compression_level', '4'],
},
h264_rkmpp: {
name: 'h264_rkmpp',
presetArgs: ['-level', '51', '-rc_mode', 'AVBR'],
},
h264_amf: {
name: 'h264_amf',
presetArgs: ['-quality', 'balanced', '-rc', 'qvbr', '-qvbr_quality_level', '26'],
@@ -254,6 +262,82 @@ export const H264_ENCODER_CONFIGS: Record<string, EncoderConfig> = {
},
}
/**
* How an encoder can also decode and scale on its own device, so frames stay in
* GPU memory instead of being decoded and scaled on the CPU and uploaded.
* Matches Immich's hardware acceleration pipelines for NVENC (CUDA), QSV, VAAPI,
* and RKMPP.
*/
export interface HardwareDecodeConfig {
/** Input options placed before `-i` (after any `-init_hw_device`). */
inputArgs: string[]
/** Dynamic input options generator taking device info if applicable. */
getInputArgs?: (driDevice?: string) => string[]
/** Output options placed after the filter graph. */
outputArgs: string[]
/** GPU filter fitting the frame within width x height with even dimensions. */
scaleFilter: (width: number, height: number) => string
}
export function resolveHwDecodeInputArgs(
config: HardwareDecodeConfig,
driDevice?: string,
): string[] {
return config.getInputArgs ? config.getInputArgs(driDevice) : config.inputArgs
}
export const HW_DECODE_CONFIGS: Partial<Record<string, HardwareDecodeConfig>> = {
h264_nvenc: {
inputArgs: ['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda', '-threads', '1'],
outputArgs: ['-noautoscale'],
scaleFilter: (width, height) =>
`scale_cuda=w=${width}:h=${height}:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12`,
},
h264_qsv: {
inputArgs: [
'-hwaccel',
'qsv',
'-hwaccel_output_format',
'qsv',
'-async_depth',
'4',
'-threads',
'1',
],
getInputArgs: (driDevice?: string) => [
'-hwaccel',
'qsv',
'-hwaccel_output_format',
'qsv',
'-async_depth',
'4',
'-threads',
'1',
...(driDevice ? ['-qsv_device', driDevice] : []),
],
outputArgs: ['-noautoscale'],
scaleFilter: (width, height) =>
`scale_qsv=w=${width}:h=${height}:async_depth=4:mode=hq:format=nv12`,
},
h264_vaapi: {
inputArgs: ['-hwaccel', 'vaapi', '-hwaccel_device', 'accel', '-hwaccel_output_format', 'vaapi'],
// If the decoder reinitializes mid-stream with a different coded size
// (e.g. 1080 -> 1088), ffmpeg would otherwise auto-insert a software
// scaler, which cannot take GPU frames, and fail.
outputArgs: ['-noautoscale'],
// Output range is left as-is (matching the software path): on i965,
// out_range only retags the stream without converting levels.
scaleFilter: (width, height) =>
`scale_vaapi=w=${width}:h=${height}:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12:mode=hq`,
},
h264_rkmpp: {
inputArgs: ['-hwaccel', 'rkmpp', '-hwaccel_output_format', 'drm_prime', '-afbc', 'rga'],
outputArgs: ['-noautoscale'],
scaleFilter: (width, height) =>
`scale_rkrga=w=${width}:h=${height}:format=nv12:afbc=1:async_depth=4`,
},
}
export function getPlatformEncoderCandidates(
platform: NodeJS.Platform = process.platform,
): string[] {
@@ -264,12 +348,13 @@ export function getPlatformEncoderCandidates(
return ['h264_nvenc', 'h264_qsv', 'h264_amf']
case 'linux':
default:
return ['h264_nvenc', 'h264_vaapi', 'h264_qsv', 'h264_amf']
return ['h264_nvenc', 'h264_vaapi', 'h264_qsv', 'h264_rkmpp', 'h264_amf']
}
}
export function getVaapiDevice(driDir = '/dev/dri'): string | null {
const envDevice = process.env.SHUMAI_VAAPI_DEVICE || process.env.VAAPI_DEVICE
export function getDriDevice(driDir = '/dev/dri'): string | null {
const envDevice =
process.env.SHUMAI_HW_DEVICE || process.env.SHUMAI_VAAPI_DEVICE || process.env.VAAPI_DEVICE
if (envDevice) {
return envDevice
}
@@ -293,6 +378,47 @@ export function getVaapiDevice(driDir = '/dev/dri'): string | null {
return null
}
export function getVaapiDevice(driDir = '/dev/dri'): string | null {
return getDriDevice(driDir)
}
/**
* Whether hardware encoders with a {@link HW_DECODE_CONFIGS} entry also decode
* and scale on the GPU. Enabled by default; set SHUMAI_HW_DECODE=false to disable.
*/
export function isHwDecodeEnabled(): boolean {
const value = process.env.SHUMAI_HW_DECODE?.trim().toLowerCase()
return value !== 'false' && value !== '0' && value !== 'off'
}
/**
* A GPU-decode attempt whose output stops growing for this long is killed and
* retried with software decode, since some driver failures hang ffmpeg instead
* of exiting. Override with SHUMAI_HW_DECODE_STALL_TIMEOUT (seconds).
*/
export function getHwDecodeStallTimeoutMs(): number {
const seconds = parseFloat(process.env.SHUMAI_HW_DECODE_STALL_TIMEOUT ?? '')
return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : 120_000
}
function getPathSize(target: string): number {
try {
const stat = fs.statSync(target)
if (!stat.isDirectory()) {
return stat.size
}
return fs
.readdirSync(target)
.reduce((total, entry) => total + getPathSize(path.join(target, entry)), 0)
} catch {
return 0
}
}
function normalizeRotation(rotation: number): number {
return ((Math.round(rotation) % 360) + 360) % 360
}
export function parseBitrateKbps(bitrate: string | number): number {
if (typeof bitrate === 'number') {
return Math.round(bitrate / 1000)
@@ -784,7 +910,7 @@ export class TranscodeService {
}
}
// RAW branch — extract embedded JPEG for metadata
// RAW branch — extract embedded JPEG or decode RAW for metadata
if (typeof input === 'string' && isRawImage(input)) {
const extracted = await extractAndValidateRawPreview(input)
if (extracted) {
@@ -793,8 +919,10 @@ export class TranscodeService {
extracted.orientation !== undefined &&
extracted.orientation >= 5 &&
extracted.orientation <= 8
const originalWidth = isSwapped ? extracted.height : extracted.width
const originalHeight = isSwapped ? extracted.width : extracted.height
const sourceWidth = extracted.rawWidth ?? extracted.width
const sourceHeight = extracted.rawHeight ?? extracted.height
const originalWidth = isSwapped ? sourceHeight : sourceWidth
const originalHeight = isSwapped ? sourceWidth : sourceHeight
return {
originalWidth,
@@ -898,8 +1026,12 @@ export class TranscodeService {
}
}
getDriDevice(driDir = '/dev/dri'): string | null {
return this.getVaapiDevice(driDir)
}
getVaapiDevice(driDir = '/dev/dri'): string | null {
return getVaapiDevice(driDir)
return getDriDevice(driDir)
}
async isEncoderUsable(encoder: string): Promise<boolean> {
@@ -1007,6 +1139,31 @@ export class TranscodeService {
async transcodeVideo(params: TranscodeVideoParams): Promise<void> {
const encoder = await this.selectH264Encoder(params.hardwareAcceleration)
if (await this.canUseHwDecode(params, encoder)) {
try {
await this.executeFfmpegVideoTranscode(params, encoder, { hwDecode: true })
return
} catch (err) {
if (this.isAbortError(err, params.signal)) {
throw err
}
logger.warn(
{
err,
encoder: encoder.name,
inputFile: params.inputFile,
outputFile: params.outputFile,
width: params.width,
height: params.height,
},
'Hardware decode failed; retrying with software decode and hardware encode',
)
this.removeFileIfExists(params.outputFile)
}
}
if (encoder.name !== 'libx264') {
try {
await this.executeFfmpegVideoTranscode(params, encoder)
@@ -1028,13 +1185,7 @@ export class TranscodeService {
'Hardware video transcoding failed; falling back to software transcode (libx264)',
)
if (fs.existsSync(params.outputFile)) {
try {
fs.unlinkSync(params.outputFile)
} catch {
// ignore unlink errors
}
}
this.removeFileIfExists(params.outputFile)
await this.executeFfmpegVideoTranscode(params, H264_ENCODER_CONFIGS.libx264)
return
@@ -1044,9 +1195,217 @@ export class TranscodeService {
await this.executeFfmpegVideoTranscode(params, encoder)
}
/**
* GPU decode + scale is attempted only for plain proxies: watermark overlays,
* HDR sources and rotated sources keep the software decode path. (ffmpeg
* cannot auto-insert its rotation filter on GPU frames, so a rotated phone
* clip would otherwise come out sideways rather than fail.) Sources the GPU
* cannot decode (e.g. HEVC/10-bit on older Intel iGPUs) fail fast and are
* retried with software decode by the caller.
*/
private async canUseHwDecode(
params: Pick<
TranscodeVideoParams,
| 'inputFile'
| 'streamIndex'
| 'overlayFile'
| 'sourceIsHdr'
| 'sourceHdrType'
| 'sourceRotation'
>,
encoder: EncoderConfig,
): Promise<boolean> {
if (!HW_DECODE_CONFIGS[encoder.name] || !isHwDecodeEnabled()) {
return false
}
if (params.overlayFile) {
return false
}
const isSourceHdr =
params.sourceIsHdr ||
params.sourceHdrType === 'pq' ||
params.sourceHdrType === 'hlg' ||
params.sourceHdrType === 'dovi_p5'
if (isSourceHdr) {
return false
}
if (encoder.name === 'h264_vaapi' && !this.getDriDevice()) {
return false
}
if (encoder.name === 'h264_qsv' && process.platform === 'linux' && !this.getDriDevice()) {
return false
}
const rotation =
params.sourceRotation !== undefined
? normalizeRotation(params.sourceRotation)
: await this.getVideoRotation(params.inputFile, params.streamIndex)
return rotation === 0
}
/**
* Rotation of the source video stream in degrees (0, 90, 180 or 270), from
* the display matrix or the legacy `rotate` tag. Returns null if it cannot be
* determined.
*/
async getVideoRotation(inputFile: string, streamIndex?: number): Promise<number | null> {
try {
const { stdout } = await execFileAsync('ffprobe', [
'-v',
'error',
'-select_streams',
streamIndex !== undefined ? String(streamIndex) : 'V:0',
'-show_entries',
'stream_side_data=rotation:stream_tags=rotate',
'-of',
'json',
inputFile,
])
const stream = JSON.parse(stdout)?.streams?.[0]
if (!stream) {
return null
}
let rotation = 0
const sideData = Array.isArray(stream.side_data_list) ? stream.side_data_list : []
const matrix = sideData.find((d: { rotation?: unknown }) => typeof d.rotation === 'number')
if (matrix) {
rotation = matrix.rotation
} else if (stream.tags?.rotate !== undefined) {
rotation = parseFloat(stream.tags.rotate) || 0
}
return normalizeRotation(rotation)
} catch (err) {
logger.debug({ err, inputFile }, 'Failed to probe video rotation')
return null
}
}
private removeFileIfExists(filePath: string): void {
if (fs.existsSync(filePath)) {
try {
fs.unlinkSync(filePath)
} catch {
// ignore unlink errors
}
}
}
/**
* Inputs, video filter graph and `-map [vout]`, shared by the MP4 and HLS
* paths: GPU decode + GPU scale when `hwDecodeConfig` is set, otherwise
* software decode + scale (+ upload for VAAPI encode).
*/
private buildVideoInputArgs(
params: Pick<
TranscodeVideoParams,
'inputFile' | 'overlayFile' | 'streamIndex' | 'width' | 'height'
>,
options: {
isVaapi: boolean
driDevice?: string
hwDecodeConfig?: HardwareDecodeConfig
frameRate?: number | string
},
): string[] {
const { isVaapi, driDevice, hwDecodeConfig, frameRate } = options
const args: string[] = []
if (isVaapi && driDevice) {
args.push('-init_hw_device', `vaapi=accel:${driDevice}`, '-filter_hw_device', 'accel')
}
if (hwDecodeConfig) {
args.push(...resolveHwDecodeInputArgs(hwDecodeConfig, driDevice))
}
args.push('-i', params.inputFile)
const baseScale = hwDecodeConfig
? hwDecodeConfig.scaleFilter(params.width, params.height)
: `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
let filterComplex: string
if (params.overlayFile) {
args.push('-i', params.overlayFile)
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
} else {
filterComplex = `[${vPad}]${baseScale}`
}
if (frameRate) {
filterComplex += `,fps=${frameRate}`
}
if (isVaapi && !hwDecodeConfig) {
filterComplex += ',format=nv12,hwupload=extra_hw_frames=64'
}
filterComplex += '[vout]'
args.push('-filter_complex', filterComplex, '-map', '[vout]')
if (hwDecodeConfig) {
args.push(...hwDecodeConfig.outputArgs)
}
return args
}
/**
* Runs ffmpeg. With `stallWatchPath`, ffmpeg is killed (SIGKILL; hung VAAPI
* processes have been seen to ignore SIGTERM) if that output file/directory
* stops growing for {@link getHwDecodeStallTimeoutMs}, and the call rejects
* with a regular error so the caller can fall back.
*/
private async runFfmpeg(
args: string[],
options: { signal?: AbortSignal; stallWatchPath?: string } = {},
): Promise<void> {
const ffmpegArgs = ['-y', '-loglevel', 'warning', ...args]
if (!options.stallWatchPath) {
if (options.signal) {
await execFileAsync('ffmpeg', ffmpegArgs, { signal: options.signal })
} else {
await execFileAsync('ffmpeg', ffmpegArgs)
}
return
}
const stallTimeoutMs = getHwDecodeStallTimeoutMs()
const stall = new AbortController()
const signal = options.signal ? AbortSignal.any([options.signal, stall.signal]) : stall.signal
const watchPath = options.stallWatchPath
let lastSize = -1
let lastProgressAt = Date.now()
const timer = setInterval(
() => {
const size = getPathSize(watchPath)
if (size !== lastSize) {
lastSize = size
lastProgressAt = Date.now()
} else if (Date.now() - lastProgressAt >= stallTimeoutMs) {
stall.abort()
}
},
Math.min(5000, stallTimeoutMs),
)
timer.unref?.()
try {
await execFileAsync('ffmpeg', ffmpegArgs, { signal, killSignal: 'SIGKILL' })
} catch (err) {
if (stall.signal.aborted && !options.signal?.aborted) {
throw new Error(
`ffmpeg made no progress for ${Math.round(stallTimeoutMs / 1000)}s and was killed`,
{ cause: err },
)
}
throw err
} finally {
clearInterval(timer)
}
}
private async executeFfmpegVideoTranscode(
params: TranscodeVideoParams,
encoder: EncoderConfig,
options: { hwDecode?: boolean } = {},
): Promise<void> {
const isSourceHdr =
params.sourceIsHdr ||
@@ -1056,7 +1415,9 @@ export class TranscodeService {
const isHdrOutput = Boolean(params.hdr)
const isVaapi = encoder.name === 'h264_vaapi'
const vaapiDevice = isVaapi ? (this.getVaapiDevice() ?? '/dev/dri/renderD128') : undefined
const driDevice = this.getDriDevice() ?? (isVaapi ? '/dev/dri/renderD128' : undefined)
const hwDecodeConfig = options.hwDecode ? HW_DECODE_CONFIGS[encoder.name] : undefined
const hwDecode = Boolean(hwDecodeConfig)
logger.info(
{
@@ -1064,39 +1425,19 @@ export class TranscodeService {
hardwareAcceleration: params.hardwareAcceleration ?? 'off',
inputFile: params.inputFile,
outputFile: params.outputFile,
vaapiDevice,
vaapiDevice: isVaapi ? driDevice : undefined,
driDevice,
hwDecode,
},
'Starting video transcoding',
)
let filterComplex: string
const args: string[] = []
if (isVaapi && vaapiDevice) {
args.push('-init_hw_device', `vaapi=accel:${vaapiDevice}`, '-filter_hw_device', 'accel')
}
args.push('-i', params.inputFile)
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
if (params.overlayFile) {
args.push('-i', params.overlayFile)
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
} else {
filterComplex = `[${vPad}]${baseScale}`
}
if (params.frameRate) {
filterComplex += `,fps=${params.frameRate}`
}
if (isVaapi) {
filterComplex += ',format=nv12,hwupload=extra_hw_frames=64'
}
filterComplex += '[vout]'
args.push('-filter_complex', filterComplex, '-map', '[vout]')
const args = this.buildVideoInputArgs(params, {
isVaapi,
driDevice,
hwDecodeConfig,
frameRate: params.frameRate,
})
if (params.frameRate) {
let calculatedFps: number
@@ -1171,6 +1512,18 @@ export class TranscodeService {
'-minrate',
`${minKbps}k`,
)
} else if (encoder.name === 'h264_rkmpp') {
if (params.videoBitrate) {
args.push('-b:v', params.videoBitrate)
} else {
const { maxrate } = calculateMaxBitrate(
params.height,
params.width,
params.sourceVideoBitrate,
params.frameRate,
)
args.push('-b:v', maxrate)
}
} else {
if (params.videoBitrate) {
args.push('-b:v', params.videoBitrate)
@@ -1185,7 +1538,7 @@ export class TranscodeService {
}
}
if (!isVaapi) {
if (!hwDecode && !isVaapi) {
args.push('-pix_fmt', 'yuv420p')
}
@@ -1208,18 +1561,40 @@ export class TranscodeService {
args.push('-movflags', '+faststart', '-max_muxing_queue_size', '1024', params.outputFile)
if (params.signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
signal: params.signal,
})
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
await this.runFfmpeg(args, {
signal: params.signal,
stallWatchPath: hwDecode ? params.outputFile : undefined,
})
}
async transcodeHlsRendition(params: TranscodeHlsRenditionParams): Promise<void> {
const encoder = await this.selectH264Encoder(params.hardwareAcceleration)
if (await this.canUseHwDecode(params, encoder)) {
try {
await this.executeFfmpegHlsTranscode(params, encoder, { hwDecode: true })
return
} catch (err) {
if (this.isAbortError(err, params.signal)) {
throw err
}
logger.warn(
{
err,
encoder: encoder.name,
inputFile: params.inputFile,
outputDir: params.outputDir,
width: params.width,
height: params.height,
},
'Hardware decode failed for HLS; retrying with software decode and hardware encode',
)
this.removeDirIfExists(params.outputDir)
}
}
if (encoder.name !== 'libx264') {
try {
await this.executeFfmpegHlsTranscode(params, encoder)
@@ -1241,13 +1616,7 @@ export class TranscodeService {
'Hardware HLS transcoding failed; falling back to software transcode (libx264)',
)
if (fs.existsSync(params.outputDir)) {
try {
fs.rmSync(params.outputDir, { recursive: true, force: true })
} catch {
// ignore unlink errors
}
}
this.removeDirIfExists(params.outputDir)
await this.executeFfmpegHlsTranscode(params, H264_ENCODER_CONFIGS.libx264)
return
@@ -1257,9 +1626,20 @@ export class TranscodeService {
await this.executeFfmpegHlsTranscode(params, encoder)
}
private removeDirIfExists(dirPath: string): void {
if (fs.existsSync(dirPath)) {
try {
fs.rmSync(dirPath, { recursive: true, force: true })
} catch {
// ignore unlink errors
}
}
}
private async executeFfmpegHlsTranscode(
params: TranscodeHlsRenditionParams,
encoder: EncoderConfig,
options: { hwDecode?: boolean } = {},
): Promise<void> {
fs.mkdirSync(params.outputDir, { recursive: true })
@@ -1271,7 +1651,9 @@ export class TranscodeService {
const isHdrOutput = Boolean(params.hdr)
const isVaapi = encoder.name === 'h264_vaapi'
const vaapiDevice = isVaapi ? (this.getVaapiDevice() ?? '/dev/dri/renderD128') : undefined
const driDevice = this.getDriDevice() ?? (isVaapi ? '/dev/dri/renderD128' : undefined)
const hwDecodeConfig = options.hwDecode ? HW_DECODE_CONFIGS[encoder.name] : undefined
const hwDecode = Boolean(hwDecodeConfig)
const segmentDuration = params.segmentDuration || 4
let calculatedFps = 30
@@ -1302,37 +1684,17 @@ export class TranscodeService {
height: params.height,
segmentDuration,
gopSize,
hwDecode,
},
'Starting HLS rendition transcoding',
)
let filterComplex: string
const args: string[] = []
if (isVaapi && vaapiDevice) {
args.push('-init_hw_device', `vaapi=accel:${vaapiDevice}`, '-filter_hw_device', 'accel')
}
args.push('-i', params.inputFile)
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
if (params.overlayFile) {
args.push('-i', params.overlayFile)
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
} else {
filterComplex = `[${vPad}]${baseScale}`
}
filterComplex += `,fps=${calculatedFps}`
if (isVaapi) {
filterComplex += ',format=nv12,hwupload=extra_hw_frames=64'
}
filterComplex += '[vout]'
args.push('-filter_complex', filterComplex, '-map', '[vout]')
const args = this.buildVideoInputArgs(params, {
isVaapi,
driDevice,
hwDecodeConfig,
frameRate: calculatedFps,
})
if (!params.disableAudio) {
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
@@ -1396,7 +1758,7 @@ export class TranscodeService {
maxKbps = parseInt(maxrate, 10)
}
const targetKbps = Math.max(25, Math.ceil(maxKbps / 1.45))
args.push('-b:v', `${targetKbps}k`, '-maxrate', `${maxKbps}k`)
args.push('-b:v', `${targetKbps}k`, '-maxrate', `${maxKbps}k`, '-idr_interval', '0')
} else if (encoder.name === 'h264_nvenc') {
let maxrateStr: string
let bufsizeStr: string
@@ -1427,7 +1789,23 @@ export class TranscodeService {
'1',
'-temporal-aq',
'1',
'-forced-idr',
'1',
)
} else if (encoder.name === 'h264_rkmpp') {
let maxKbps: number
if (params.videoBitrate) {
maxKbps = parseBitrateKbps(params.videoBitrate)
} else {
const { maxrate } = calculateMaxBitrate(
params.height,
params.width,
params.sourceVideoBitrate,
calculatedFps,
)
maxKbps = parseInt(maxrate, 10)
}
args.push('-b:v', `${maxKbps}k`)
} else {
let maxrateStr: string
let bufsizeStr: string
@@ -1448,7 +1826,7 @@ export class TranscodeService {
args.push('-crf', '23', '-maxrate', maxrateStr, '-bufsize', bufsizeStr)
}
if (!isVaapi) {
if (!hwDecode && !isVaapi) {
args.push('-pix_fmt', 'yuv420p')
}
@@ -1506,13 +1884,10 @@ export class TranscodeService {
playlistFile,
)
if (params.signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
signal: params.signal,
})
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
await this.runFfmpeg(args, {
signal: params.signal,
stallWatchPath: hwDecode ? params.outputDir : undefined,
})
}
async transcodeImage(
@@ -1545,13 +1920,15 @@ export class TranscodeService {
input = Buffer.from(await resp.arrayBuffer())
}
// RAW branch — extract embedded JPEG to temporary file + orientation, then treat as normal image
// RAW branch — extract embedded preview or decode RAW to temporary file + orientation, then treat as normal image
let rawOrientation: number | undefined
let rawCleanup: (() => void) | null = null
if (typeof input === 'string' && isRawImage(input)) {
const extracted = await extractAndValidateRawPreview(input)
if (!extracted) {
throw new Error(`Cannot generate preview for RAW file: no usable embedded JPEG in ${input}`)
throw new Error(
`Cannot generate preview for RAW file: no usable embedded preview or decoded image in ${input}`,
)
}
input = extracted.previewPath
rawOrientation = extracted.orientation
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/db",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/dtos",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/e2e",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"dependencies": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/tableprinter",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/transcode",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
@@ -998,6 +998,24 @@ describe('Transcode Activities', () => {
).rejects.toThrowError(/Image transcoding failed/)
})
it('should throw non-retryable ApplicationFailure when transcodeImageActivity fails with dcraw error', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
vi.mocked(transcodeService.transcodeImage).mockRejectedValue(
new Error('dcraw_emu failed to decode RAW file'),
)
await expect(
transcodeImageActivity({
assetKey: 'photo.cr2',
filePath: '/tmp/photo.cr2',
imageSpec: { width: 800, height: 600, quality: 90, format: 'webp' },
}),
).rejects.toMatchObject({
message: expect.stringContaining('Image transcoding failed'),
nonRetryable: true,
})
})
it('should throw non-retryable ApplicationFailure when generateSpriteActivity fails with error', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
vi.mocked(transcodeService.generateSprite).mockRejectedValue(new Error('FFmpeg failed'))
@@ -334,6 +334,7 @@ export interface VideoActivityParams {
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
sourceRotation?: number
streamIndex?: number
audioStreamIndex?: number
}
@@ -397,6 +398,7 @@ export async function transcodeVideoActivity(
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
sourceRotation: params.sourceRotation,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
@@ -464,6 +466,7 @@ export interface HlsActivityParams {
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
sourceRotation?: number
streamIndex?: number
audioStreamIndex?: number
}
@@ -532,6 +535,7 @@ export async function transcodeHlsActivity(params: HlsActivityParams): Promise<P
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
sourceRotation: params.sourceRotation,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
@@ -772,6 +776,8 @@ export async function transcodeImageActivity(
lowerMsg.includes('imagemagick') ||
lowerMsg.includes('magick') ||
lowerMsg.includes('convert') ||
lowerMsg.includes('dcraw') ||
lowerMsg.includes('raw') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format')
) {
@@ -167,6 +167,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
sourceColorTransfer: metadata.colorTransfer,
sourceColorPrimaries: metadata.colorPrimaries,
sourceColorSpace: metadata.colorSpace,
sourceRotation: metadata.rotation ?? 0,
streamIndex: metadata.videoStreamIndex,
audioStreamIndex: metadata.audioStreamIndex,
})
@@ -197,6 +198,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
sourceColorTransfer: metadata.colorTransfer,
sourceColorPrimaries: metadata.colorPrimaries,
sourceColorSpace: metadata.colorSpace,
// rotation is undefined when the probed source has none
sourceRotation: metadata.rotation ?? 0,
streamIndex: metadata.videoStreamIndex,
audioStreamIndex: metadata.audioStreamIndex,
})
@@ -227,6 +230,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
sourceColorTransfer: metadata.colorTransfer,
sourceColorPrimaries: metadata.colorPrimaries,
sourceColorSpace: metadata.colorSpace,
sourceRotation: metadata.rotation ?? 0,
streamIndex: metadata.videoStreamIndex,
audioStreamIndex: metadata.audioStreamIndex,
})
@@ -136,13 +136,19 @@ export const NotificationCard = ({ notification }: NotificationCardProps) => {
const hasPreview = asset?.preview && (asset.proxyType === 'image' || asset.proxyType === 'video')
const handleClick = () => {
const handleClick = () => {
if (kanbanTask?.id && team?.id) {
navigate({
to: '/teams/$teamId/kanban',
params: { teamId: team.id },
search: { taskId: kanbanTask.id },
})
}else if(project?.id && asset?.id){
//TODO: Perhaps it should be marked as read too when redirecting to it.
navigate({
to: '/projects/$projectId/files/$fileId',
params: { projectId: project.id, fileId: asset.id },
})
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/webui",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./index.html",
+1 -1
View File
@@ -1 +1 @@
export const APP_VERSION = '0.5.0'
export const APP_VERSION = '0.5.1'
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@shumai/workflow-core",
"version": "0.5.0",
"version": "0.5.1",
"private": true,
"type": "module",
"main": "./src/index.ts",
+2 -2
View File
@@ -70,7 +70,7 @@ export function getActivities<T = any>(): T {
// Temporal's proxyActivities requires a type parameter, so we use any for generic activities.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
baseProxy = wf.proxyActivities<any>({
startToCloseTimeout: '10 minutes',
startToCloseTimeout: '120 minutes',
retry: {
maximumAttempts: 5,
initialInterval: '10s',
@@ -165,7 +165,7 @@ export async function executeActivity(
if (isTemporal()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const handle = wf.proxyActivities<any>({
startToCloseTimeout: '10 minutes',
startToCloseTimeout: '120 minutes',
taskQueue: queue,
retry: {
maximumAttempts: 5,
+1
View File
@@ -13,6 +13,7 @@ const depsToExtract = [
'@temporalio/client',
'@temporalio/worker',
'@temporalio/workflow',
'exiftool-vendored',
'pdfkit',
'prisma',
'sharp',