From 3564d4d164b6933efdb7b0118f3731c209f2935b Mon Sep 17 00:00:00 2001 From: Yiling-J Date: Fri, 2 Oct 2026 13:51:17 +0800 Subject: [PATCH] fix(transcode): avoid selecting attached picture streams for video transcoding (#443) * fix(transcode): avoid selecting attached picture streams for video transcoding Filter out attached picture streams (disposition.attached_pic) during video probing and stream selection, matching Immich's default and bitrate selection strategy. Forward explicit primary video and audio stream indices to FFmpeg pipelines to prevent picking embedded cover art or thumbnail streams. * fix(transcode): skip poster and sprite generation for audio-only assets and add multi-stream e2e tests * test(workflow): scope pixel format to h264 stream and use yuvj420p for mjpeg cover art --- CHANGELOG.md | 2 + packages/agent/src/activities/ai.ts | 4 + packages/core/src/transcode/transcode.test.ts | 192 ++++- packages/core/src/transcode/transcode.ts | 151 +++- packages/db/src/prisma-json-types.ts | 2 + packages/e2e/workflow/transcode-video.test.ts | 740 ++++++++++++++++++ .../src/activities/transcode.test.ts | 140 ++++ .../transcode/src/activities/transcode.ts | 166 ++-- .../transcode/src/activities/watermark.ts | 4 + .../src/workflows/transcode-video.ts | 18 +- 10 files changed, 1346 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5e37505..442345ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +* **transcode**: Fix an issue where videos or audio-only files containing attached cover art or picture streams could cause transcoding to fail during poster generation or select a single static image instead of the video + * **webui**: Fix inconsistent video resolution labels across download menus, mobile file headers, and the video player by correctly deriving resolutions from the video's short side, including portrait videos and 480p videos * **trash**: Fix an issue where the "Empty Trash" confirmation dialog would close immediately instead of showing a loading state while files were being permanently deleted diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index d6ad6f52..85f8befe 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -154,6 +154,10 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams) start.toString(), '-i', inputSource, + '-map', + '0:V:0', + '-map', + '0:a:0?', '-t', (end - start).toString(), '-c', diff --git a/packages/core/src/transcode/transcode.test.ts b/packages/core/src/transcode/transcode.test.ts index 112f8169..790f44fc 100644 --- a/packages/core/src/transcode/transcode.test.ts +++ b/packages/core/src/transcode/transcode.test.ts @@ -12,6 +12,9 @@ import { parseBitrateKbps, buildHlsMasterPlaylist, rewriteM3u8WithPresignedUrls, + compareStreams, + selectPrimaryVideoStream, + selectPrimaryAudioStream, } from './transcode' import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' @@ -712,6 +715,8 @@ describe('TranscodeService', () => { '-i', 'input.wav', '-vn', + '-map', + '0:a:0?', '-c:a', 'aac', '-b:a', @@ -746,6 +751,8 @@ describe('TranscodeService', () => { '-i', 'input.wav', '-vn', + '-map', + '0:a:0?', '-c:a', 'aac', '-b:a', @@ -782,6 +789,8 @@ describe('TranscodeService', () => { '-i', 'input.wav', '-vn', + '-map', + '0:a:0?', '-c:a', 'aac', '-b:a', @@ -2831,6 +2840,187 @@ describe('TranscodeService', () => { expect(info.rotation).toBe(180) }) + it('getVideoInfo ignores attached picture streams and selects the main video stream', async () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const mockOutput = JSON.stringify({ + format: { duration: '10.0', bit_rate: '20000000' }, + streams: [ + { + index: 0, + codec_type: 'video', + codec_name: 'mjpeg', + width: 320, + height: 240, + avg_frame_rate: '0/0', + r_frame_rate: '90000/1', + disposition: { + attached_pic: 1, + }, + }, + { + index: 1, + codec_type: 'video', + codec_name: 'h264', + width: 1920, + height: 1080, + avg_frame_rate: '24/1', + r_frame_rate: '24/1', + nb_frames: '240', + disposition: { + default: 1, + attached_pic: 0, + }, + }, + ], + }) + /* eslint-enable @typescript-eslint/naming-convention */ + + mockExecFileStdout(mockOutput) + + const info = await transcodeService.getVideoInfo('cover-art-first.mp4') + expect(info.originalWidth).toBe(1920) + expect(info.originalHeight).toBe(1080) + expect(info.videoStreamIndex).toBe(1) + }) + + it('getVideoInfo throws error when file only contains audio and attached picture', async () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const mockOutput = JSON.stringify({ + format: { duration: '10.0', bit_rate: '320000' }, + streams: [ + { + index: 0, + codec_type: 'audio', + codec_name: 'aac', + channels: 2, + sample_rate: '44100', + }, + { + index: 1, + codec_type: 'video', + codec_name: 'mjpeg', + width: 320, + height: 240, + disposition: { + attached_pic: 1, + }, + }, + ], + }) + /* eslint-enable @typescript-eslint/naming-convention */ + + mockExecFileStdout(mockOutput) + + await expect(transcodeService.getVideoInfo('audio-with-cover.mp4')).rejects.toThrow( + 'No video stream found', + ) + }) + + it('compareStreams prioritizes default stream then highest bitrate', () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const s1 = { index: 0, bit_rate: '5000000', disposition: { default: 0 } } + const s2 = { index: 1, bit_rate: '2000000', disposition: { default: 1 } } + const s3 = { index: 2, bit_rate: '8000000', disposition: { default: 0 } } + const s4 = { index: 3, disposition: {} } + /* eslint-enable @typescript-eslint/naming-convention */ + + const sorted = [s1, s2, s3, s4].sort(compareStreams) + expect(sorted[0].index).toBe(1) // default: 1 + expect(sorted[1].index).toBe(2) // 8000000 bps + expect(sorted[2].index).toBe(0) // 5000000 bps + expect(sorted[3].index).toBe(3) // 0 bps + }) + + it('selectPrimaryVideoStream filters out attached picture streams', () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const streams = [ + { index: 0, codec_type: 'video', disposition: { attached_pic: 1 }, bit_rate: '9999999' }, + { index: 1, codec_type: 'audio', disposition: { default: 1 } }, + { index: 2, codec_type: 'video', disposition: { default: 0 }, bit_rate: '1500000' }, + { index: 3, codec_type: 'video', disposition: { default: 1 }, bit_rate: '1000000' }, + ] + /* eslint-enable @typescript-eslint/naming-convention */ + + const selected = selectPrimaryVideoStream(streams) + expect(selected).toBeDefined() + expect(selected?.index).toBe(3) // default: 1 non-attached + }) + + it('selectPrimaryAudioStream picks default then highest bitrate audio stream', () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const streams = [ + { index: 0, codec_type: 'video', bit_rate: '2000000' }, + { index: 1, codec_type: 'audio', disposition: { default: 0 }, bit_rate: '320000' }, + { index: 2, codec_type: 'audio', disposition: { default: 1 }, bit_rate: '128000' }, + ] + /* eslint-enable @typescript-eslint/naming-convention */ + + const selected = selectPrimaryAudioStream(streams) + expect(selected).toBeDefined() + expect(selected?.index).toBe(2) // default: 1 audio + }) + + it('transcodeVideo uses explicit streamIndex and audioStreamIndex in mapping', async () => { + let executedArgs: string[] = [] + ;( + child_process.execFile as unknown as { + mockImplementation: ( + fn: ( + file: string, + args: string[], + callback: (error: Error | null, result: { stdout: string; stderr: string }) => void, + ) => void, + ) => void + } + ).mockImplementation((_file, args, callback) => { + executedArgs = args + callback(null, { stdout: '', stderr: '' }) + }) + + await transcodeService.transcodeVideo({ + inputFile: 'input.mp4', + outputFile: 'output.mp4', + width: 1280, + height: 720, + hardwareAcceleration: 'off', + streamIndex: 2, + audioStreamIndex: 1, + }) + + const filterIdx = executedArgs.indexOf('-filter_complex') + expect(filterIdx).toBeGreaterThan(-1) + expect(executedArgs[filterIdx + 1]).toContain('[0:2]scale=') + + const mapIndices = executedArgs + .map((arg, idx) => (arg === '-map' ? executedArgs[idx + 1] : null)) + .filter(Boolean) + expect(mapIndices).toContain('0:1') + }) + + it('generatePoster uses explicit streamIndex in mapping', async () => { + let executedArgs: string[] = [] + ;( + child_process.execFile as unknown as { + mockImplementation: ( + fn: ( + file: string, + args: string[], + callback: (error: Error | null, result: { stdout: string; stderr: string }) => void, + ) => void, + ) => void + } + ).mockImplementation((_file, args, callback) => { + executedArgs = args + callback(null, { stdout: '', stderr: '' }) + }) + + await transcodeService.generatePoster('input.mp4', 'poster.webp', { streamIndex: 3 }) + + const mapIdx = executedArgs.indexOf('-map') + expect(mapIdx).toBeGreaterThan(-1) + expect(executedArgs[mapIdx + 1]).toBe('0:3') + }) + it('transcodeVideo preserves HDR parameters and tags when hdr is true', async () => { let executedArgs: string[] = [] ;( @@ -3018,7 +3208,7 @@ describe('TranscodeService', () => { const filterIdx = spriteCmd!.indexOf('-filter_complex') expect(filterIdx).toBeGreaterThan(-1) const filterStr = spriteCmd![filterIdx + 1] - expect(filterStr).toContain('[0:v]fps=') + expect(filterStr).toContain('[0:V]fps=') expect(filterStr).toContain('scale=w=300:h=-2,tile=10x10[sprite_out]') }) diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts index 7321ca1e..016dfc40 100644 --- a/packages/core/src/transcode/transcode.ts +++ b/packages/core/src/transcode/transcode.ts @@ -97,6 +97,54 @@ export interface GeneratePosterOptions { hdrType?: HdrType colorTransfer?: string signal?: AbortSignal + streamIndex?: number +} + +export interface FfprobeStreamLike { + index: number + codec_type?: string + codec_name?: string + bit_rate?: string + avg_frame_rate?: string + r_frame_rate?: string + nb_frames?: string + width?: number + height?: number + channels?: number + sample_rate?: string | number + bits_per_raw_sample?: string | number + bits_per_sample?: string | number + color_transfer?: string + color_primaries?: string + color_space?: string + disposition?: Record + tags?: Record + side_data_list?: Array> + mime_codec_string?: string + [key: string]: unknown +} + +/** + * Immich-compatible stream comparator: + * 1. Streams with disposition.default: 1 come first. + * 2. Ties are broken by higher bit_rate. + */ +export function compareStreams(a: T, b: T): number { + const defDiff = (b.disposition?.default ?? 0) - (a.disposition?.default ?? 0) + if (defDiff !== 0) return defDiff + const aBitrate = parseInt(a.bit_rate || '0', 10) || 0 + const bBitrate = parseInt(b.bit_rate || '0', 10) || 0 + return bBitrate - aBitrate +} + +export function selectPrimaryVideoStream(streams: T[]): T | undefined { + return streams + .filter((s) => s.codec_type === 'video' && !s.disposition?.attached_pic) + .sort(compareStreams)[0] +} + +export function selectPrimaryAudioStream(streams: T[]): T | undefined { + return streams.filter((s) => s.codec_type === 'audio').sort(compareStreams)[0] } export interface MediaMetadata { @@ -123,6 +171,8 @@ export interface MediaMetadata { dvProfile?: number dvCompatibilityId?: number rotation?: number + videoStreamIndex?: number + audioStreamIndex?: number } export interface TranscodeVideoParams { @@ -144,6 +194,8 @@ export interface TranscodeVideoParams { sourceColorTransfer?: string sourceColorPrimaries?: string sourceColorSpace?: string + streamIndex?: number + audioStreamIndex?: number } export interface TranscodeHlsRenditionParams { @@ -166,6 +218,8 @@ export interface TranscodeHlsRenditionParams { sourceColorPrimaries?: string sourceColorSpace?: string segmentDuration?: number + streamIndex?: number + audioStreamIndex?: number } export interface EncoderConfig { @@ -475,10 +529,15 @@ export class TranscodeService { return stream.codec_name } - private safeParseInt(value: string | number | undefined): number | undefined { + private safeParseInt(value: unknown): number | undefined { if (value === undefined || value === null) return undefined - const parsed = typeof value === 'string' ? parseInt(value, 10) : value - return Number.isFinite(parsed) ? (parsed as number) : undefined + const parsed = + typeof value === 'string' + ? parseInt(value, 10) + : typeof value === 'number' + ? value + : undefined + return parsed !== undefined && Number.isFinite(parsed) ? parsed : undefined } async getVideoInfo(inputFile: string): Promise { @@ -492,10 +551,8 @@ export class TranscodeService { inputFile, ]) const info = JSON.parse(stdout) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const videoStream = info.streams.find((s: any) => s.codec_type === 'video') - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const audioStream = info.streams.find((s: any) => s.codec_type === 'audio') + const videoStream = selectPrimaryVideoStream(info.streams || []) + const audioStream = selectPrimaryAudioStream(info.streams || []) if (!videoStream) { throw new Error('No video stream found') @@ -678,6 +735,8 @@ export class TranscodeService { hdrType, dvProfile, dvCompatibilityId, + videoStreamIndex: videoStream.index, + audioStreamIndex: audioStream ? audioStream.index : undefined, } } @@ -1020,11 +1079,13 @@ export class TranscodeService { args.push('-i', params.inputFile) const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'` + const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V' + if (params.overlayFile) { args.push('-i', params.overlayFile) - filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` + filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` } else { - filterComplex = `[0:v]${baseScale}` + filterComplex = `[${vPad}]${baseScale}` } if (params.frameRate) { @@ -1063,7 +1124,8 @@ export class TranscodeService { } if (!params.disableAudio) { - args.push('-map', '0:a?') + const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?' + args.push('-map', aMap) } args.push('-c:v', encoder.name) @@ -1254,11 +1316,13 @@ export class TranscodeService { args.push('-i', params.inputFile) const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'` + const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V' + if (params.overlayFile) { args.push('-i', params.overlayFile) - filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` + filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` } else { - filterComplex = `[0:v]${baseScale}` + filterComplex = `[${vPad}]${baseScale}` } filterComplex += `,fps=${calculatedFps}` @@ -1271,7 +1335,8 @@ export class TranscodeService { args.push('-filter_complex', filterComplex, '-map', '[vout]') if (!params.disableAudio) { - args.push('-map', '0:a?') + const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?' + args.push('-map', aMap) } args.push('-c:v', encoder.name) @@ -1624,6 +1689,8 @@ export class TranscodeService { filters.push('scale=-2:300:force_original_aspect_ratio=decrease') + const vMap = options.streamIndex !== undefined ? `0:${options.streamIndex}` : '0:V:0' + const args = [ ...(isRemote ? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5'] @@ -1631,6 +1698,8 @@ export class TranscodeService { ...(isMpegTs ? [] : ['-skip_frame', 'nointra']), '-i', inputFile, + '-map', + vMap, '-vf', filters.join(','), '-fps_mode', @@ -1660,13 +1729,14 @@ export class TranscodeService { duration: number, signal?: AbortSignal, hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string }, + streamIndex?: number, ): Promise { if (signal?.aborted) { throw new Error('Sprite generation cancelled') } // 1. Generate poster first (fast, smart frame selection) - await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal }) + await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal, streamIndex }) // 2. Generate sprite let fileSize = Infinity @@ -1679,9 +1749,23 @@ export class TranscodeService { const isSmallAndShort = fileSize <= 50 * 1024 * 1024 && duration <= 30 if (isSmallAndShort) { - await this.generateSpriteSinglePass(inputFile, outputSprite, duration, signal, hdrOptions) + await this.generateSpriteSinglePass( + inputFile, + outputSprite, + duration, + signal, + hdrOptions, + streamIndex, + ) } else { - await this.generateSpriteSeekPool(inputFile, outputSprite, duration, signal, hdrOptions) + await this.generateSpriteSeekPool( + inputFile, + outputSprite, + duration, + signal, + hdrOptions, + streamIndex, + ) } } @@ -1691,9 +1775,11 @@ export class TranscodeService { duration: number, signal?: AbortSignal, hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string }, + streamIndex?: number, ): Promise { const spriteFps = 100 / duration let filterComplex: string + const vPad = streamIndex !== undefined ? `0:${streamIndex}` : '0:V' if (hdrOptions?.isHdr) { const availableFilters = await this.getAvailableFilters() @@ -1702,9 +1788,9 @@ export class TranscodeService { colorTransfer: hdrOptions.colorTransfer, availableFilters, }) - filterComplex = `[0:v]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` + filterComplex = `[${vPad}]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` } else { - filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` + filterComplex = `[${vPad}]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` } const args = [ @@ -1735,6 +1821,7 @@ export class TranscodeService { duration: number, signal?: AbortSignal, hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string }, + streamIndex?: number, ): Promise { if (signal?.aborted) { throw new Error('Sprite generation cancelled') @@ -1778,11 +1865,14 @@ export class TranscodeService { const ts = timestamps[idx] const framePath = path.join(tmpDir, `frame_${idx.toString().padStart(3, '0')}.webp`) + const vMap = streamIndex !== undefined ? `0:${streamIndex}` : '0:V:0' const args = [ '-ss', ts.toFixed(3), '-i', inputFile, + '-map', + vMap, '-vframes', '1', '-vf', @@ -2174,8 +2264,7 @@ export class TranscodeService { inputFile, ]) const info = JSON.parse(stdout) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const audioStream = info.streams.find((s: any) => s.codec_type === 'audio') + const audioStream = selectPrimaryAudioStream(info.streams || []) if (!audioStream) { throw new Error('No audio stream found') @@ -2199,6 +2288,7 @@ export class TranscodeService { audioBitDepth: this.safeParseInt(audioStream?.bits_per_raw_sample) ?? this.safeParseInt(audioStream?.bits_per_sample), + audioStreamIndex: audioStream.index, mimeType: '', } } @@ -2209,9 +2299,23 @@ export class TranscodeService { bitrate?: string threads?: number signal?: AbortSignal + audioStreamIndex?: number }): Promise { const bitrate = params.bitrate || '128k' - const args = ['-i', params.inputFile, '-vn', '-c:a', 'aac', '-b:a', bitrate, '-ac', '2'] + const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?' + const args = [ + '-i', + params.inputFile, + '-vn', + '-map', + aMap, + '-c:a', + 'aac', + '-b:a', + bitrate, + '-ac', + '2', + ] if (params.threads && params.threads > 0) { args.push('-threads', params.threads.toString()) } @@ -2240,6 +2344,7 @@ export class TranscodeService { const numFrames = Math.max(1, params.numFrames) const step = duration / numFrames const timestamps = Array.from({ length: numFrames }, (_, i) => i * step) + const vMap = meta.videoStreamIndex !== undefined ? `0:${meta.videoStreamIndex}` : '0:V:0' await mapConcurrent(timestamps, 10, async (t, idx) => { const outputFile = path.join(params.outputDir, `${idx + 1}.webp`) @@ -2251,6 +2356,8 @@ export class TranscodeService { t.toFixed(4), '-i', params.inputFile, + '-map', + vMap, '-vframes', '1', '-vf', @@ -2375,6 +2482,8 @@ export class TranscodeService { t.toFixed(4), '-i', inputSource, + '-map', + '0:V:0', '-vframes', '1', '-vf', diff --git a/packages/db/src/prisma-json-types.ts b/packages/db/src/prisma-json-types.ts index d667ccc6..e9d45ee8 100644 --- a/packages/db/src/prisma-json-types.ts +++ b/packages/db/src/prisma-json-types.ts @@ -170,6 +170,8 @@ declare global { isHdr?: boolean hdrType?: HdrType dvProfile?: number + videoStreamIndex?: number + audioStreamIndex?: number format: unknown } diff --git a/packages/e2e/workflow/transcode-video.test.ts b/packages/e2e/workflow/transcode-video.test.ts index bd114d65..50d36047 100644 --- a/packages/e2e/workflow/transcode-video.test.ts +++ b/packages/e2e/workflow/transcode-video.test.ts @@ -8,11 +8,56 @@ import { assetService } from '@shumai/core/src/asset/asset' import { fileURLToPath } from 'url' import * as path from 'path' import * as fs from 'fs' +import * as os from 'os' +import { execFile } from 'child_process' +import { promisify } from 'util' + +const execFileAsync = promisify(execFile) const currentDir = path.dirname(fileURLToPath(import.meta.url)) const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts') const fixturesDir = path.resolve(currentDir, '../fixtures') +async function sampleFirstFrameRgb( + buffer: Buffer, + ext: string = 'mp4', +): Promise<{ r: number; g: number; b: number }> { + const tmpFile = path.join( + os.tmpdir(), + `sample-${Date.now()}-${Math.random().toString(36).slice(2)}.${ext}`, + ) + try { + fs.writeFileSync(tmpFile, buffer) + const proc = await execFileAsync( + 'ffmpeg', + [ + '-y', + '-i', + tmpFile, + '-vframes', + '1', + '-s', + '1x1', + '-f', + 'rawvideo', + '-pix_fmt', + 'rgb24', + 'pipe:1', + ], + { encoding: 'buffer' }, + ) + return { + r: proc.stdout[0], + g: proc.stdout[1], + b: proc.stdout[2], + } + } finally { + if (fs.existsSync(tmpFile)) { + fs.unlinkSync(tmpFile) + } + } +} + describe.each(['local', 'temporal'] as const)( 'Workflow E2E - transcodeVideoWorkflow (executor: %s)', (mode) => { @@ -371,5 +416,700 @@ describe.each(['local', 'temporal'] as const)( }) expect(symlinkVariant).toContain('#EXTM3U') }, 60000) + + it('Case 1: should ignore attached picture cover art (Red) and select primary video stream (Green)', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Cover Art Video Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Cover Art Video Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case1/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case1-')) + const videoPath = path.join(tmpDir, 'case1.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=green:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:a', + '-c:v:0', + 'libx264', + '-pix_fmt:v:0', + 'yuv420p', + '-c:v:1', + 'mjpeg', + '-pix_fmt:v:1', + 'yuvj420p', + '-c:a', + 'aac', + '-disposition:v:1', + 'attached_pic', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case1/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + sprite: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + sprite: { key: string } + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('video') + expect(mediaInfo.metadata.videoStreamIndex).toBe(0) + + // Verify poster is Green (not Red attached pic) + expect(mediaInfo.poster?.key).toBeDefined() + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.g).toBeGreaterThan(100) + expect(posterColor.r).toBeLessThan(50) + expect(posterColor.b).toBeLessThan(50) + + // Verify MP4 proxy is Green + expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0) + const videoObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4') + expect(videoColor.g).toBeGreaterThan(100) + expect(videoColor.r).toBeLessThan(50) + expect(videoColor.b).toBeLessThan(50) + + // Verify sprite sheet exists + expect(mediaInfo.sprite?.key).toBeDefined() + await s3Service.headObject('shumai-e2e-test-bucket-transcode', mediaInfo.sprite.key) + }, 60000) + + it('Case 2: should select higher bitrate stream (Blue) when both streams have default flag', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Bitrate Comparison Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Bitrate Comparison Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case2/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case2-')) + const videoPath = path.join(tmpDir, 'case2.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:a', + '-c:v:0', + 'libx264', + '-b:v:0', + '200k', + '-minrate:v:0', + '200k', + '-maxrate:v:0', + '200k', + '-bufsize:v:0', + '200k', + '-nal-hrd', + 'cbr', + '-disposition:v:0', + 'default', + '-c:v:1', + 'libx264', + '-b:v:1', + '2000k', + '-minrate:v:1', + '2000k', + '-maxrate:v:1', + '2000k', + '-bufsize:v:1', + '2000k', + '-nal-hrd', + 'cbr', + '-disposition:v:1', + 'default', + '-c:a', + 'aac', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case2/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('video') + // Stream 1 (Blue, 2000k) should be selected over Stream 0 (Red, 200k) + expect(mediaInfo.metadata.videoStreamIndex).toBe(1) + + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.b).toBeGreaterThan(100) + expect(posterColor.r).toBeLessThan(50) + expect(posterColor.g).toBeLessThan(50) + + const videoObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4') + expect(videoColor.b).toBeGreaterThan(100) + expect(videoColor.r).toBeLessThan(50) + expect(videoColor.g).toBeLessThan(50) + }, 60000) + + it('Case 3: should select stream with default flag (Yellow) over higher bitrate stream without default flag (Blue)', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Default Flag Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Default Flag Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case3/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case3-')) + const videoPath = path.join(tmpDir, 'case3.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=yellow:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:a', + '-c:v:0', + 'libx264', + '-b:v:0', + '200k', + '-minrate:v:0', + '200k', + '-maxrate:v:0', + '200k', + '-bufsize:v:0', + '200k', + '-nal-hrd', + 'cbr', + '-disposition:v:0', + 'default', + '-c:v:1', + 'libx264', + '-b:v:1', + '2000k', + '-minrate:v:1', + '2000k', + '-maxrate:v:1', + '2000k', + '-bufsize:v:1', + '2000k', + '-nal-hrd', + 'cbr', + '-disposition:v:1', + '0', + '-c:a', + 'aac', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case3/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('video') + // Stream 0 (Yellow, default: 1) should win over Stream 1 (Blue, 10x higher bitrate but default: 0) + expect(mediaInfo.metadata.videoStreamIndex).toBe(0) + + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.r).toBeGreaterThan(100) + expect(posterColor.g).toBeGreaterThan(100) + expect(posterColor.b).toBeLessThan(50) + + const videoObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4') + expect(videoColor.r).toBeGreaterThan(100) + expect(videoColor.g).toBeGreaterThan(100) + expect(videoColor.b).toBeLessThan(50) + }, 60000) + + it('Case 4: should handle audio-only MP4 with cover art by downgrading to audio proxy (catches audio bug)', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Audio Cover Art Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Audio Cover Art Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case4/audio-cover.mp4' }, + }) + // Uploaded as video/mp4 because of .mp4 container, even though it only has audio + cover art + const asset = await prisma.asset.create({ + data: { + name: 'audio-cover.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case4-')) + const videoPath = path.join(tmpDir, 'case4.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=magenta:s=320x240:d=2', + '-map', + '0:a', + '-map', + '1:v', + '-c:a', + 'aac', + '-c:v', + 'mjpeg', + '-pix_fmt:v:0', + 'yuvj420p', + '-disposition:v:0', + 'attached_pic', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case4/audio-cover.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + // Spec requested poster, sprite, videoStrategy as video upload would + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + sprite: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + // Must complete without error (previously failed trying to generate poster/sprite for audio) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + poster?: unknown + sprite?: unknown + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('audio') + expect(mediaInfo.poster).toBeUndefined() + expect(mediaInfo.sprite).toBeUndefined() + expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0) + expect(mediaInfo.videoTranscodes[0].key).toContain('-audio-proxy.mp4') + await s3Service.headObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + }, 60000) + + it('Case 5: should rank 3 video streams with HLS enabled and select highest bitrate stream (Blue)', async () => { + const team = await prisma.team.create({ + data: { + name: 'E2E Multi Stream HLS Team', + settings: { + transcode: { + videoStrategy: 'best_match', + hlsEnabled: true, + hlsResolutions: ['480p'], + }, + }, + }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Multi Stream HLS Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case5/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case5-')) + const videoPath = path.join(tmpDir, 'case5.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=green:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:v', + '-map', + '3:a', + '-c:v:0', + 'libx264', + '-b:v:0', + '100k', + '-minrate:v:0', + '100k', + '-maxrate:v:0', + '100k', + '-bufsize:v:0', + '100k', + '-nal-hrd', + 'cbr', + '-disposition:v:0', + 'default', + '-c:v:1', + 'libx264', + '-b:v:1', + '500k', + '-minrate:v:1', + '500k', + '-maxrate:v:1', + '500k', + '-bufsize:v:1', + '500k', + '-nal-hrd', + 'cbr', + '-disposition:v:1', + 'default', + '-c:v:2', + 'libx264', + '-b:v:2', + '1500k', + '-minrate:v:2', + '1500k', + '-maxrate:v:2', + '1500k', + '-bufsize:v:2', + '1500k', + '-nal-hrd', + 'cbr', + '-disposition:v:2', + 'default', + '-c:a', + 'aac', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case5/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + hlsEnabled: true, + hlsResolutions: ['480p'], + poster: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + isHls: boolean + hls: { key: string; resolutions: { resolution: string }[] } + } + expect(mediaInfo.proxyType).toBe('video') + // Stream 2 (Blue, 1500k) should be selected + expect(mediaInfo.metadata.videoStreamIndex).toBe(2) + + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.b).toBeGreaterThan(100) + expect(posterColor.r).toBeLessThan(50) + expect(posterColor.g).toBeLessThan(50) + + expect(mediaInfo.isHls).toBe(true) + const masterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.hls.key, + ) + expect(masterObj.buffer.toString('utf-8')).toContain('480p/index.m3u8') + }, 60000) }, ) diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 70fdd322..3c9b19d1 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -217,6 +217,86 @@ describe('Transcode Activities', () => { ) }) + it('should downgrade proxyType to audio when getVideoInfo throws No video stream found but audio exists', async () => { + const asset = await prisma.asset.create({ + data: { name: 'audio-with-art.mp4', type: 'file', status: 'uploaded' }, + }) + + vi.mocked(transcodeService.getVideoInfo).mockRejectedValueOnce( + new Error('No video stream found'), + ) + vi.mocked(transcodeService.getAudioInfo).mockResolvedValueOnce({ + originalWidth: 0, + originalHeight: 0, + duration: 42, + bitRate: 192000, + frameRate: 0, + totalFrames: 0, + hasAudio: true, + audioCodec: 'aac', + audioChannels: 2, + audioSampleRate: 44100, + audioBitDepth: 16, + audioStreamIndex: 0, + mimeType: 'audio/mp4', + }) + + const result = await getMediaInfoActivity({ + assetId: asset.id, + filePath: '/tmp/audio-with-art.mp4', + proxyType: 'video', + mediaType: 'video/mp4', + }) + + expect(transcodeService.getVideoInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4') + expect(transcodeService.getAudioInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4') + expect(result.proxyType).toBe('audio') + expect(result.duration).toBe(42) + expect(result.metadata?.hasAudio).toBe(true) + expect(result.metadata?.audioStreamIndex).toBe(0) + expect(result.metadata?.videoStreamIndex).toBeUndefined() + expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith( + asset.id, + expect.arrayContaining([ + { key: 'file_type', value: 'audio' }, + { key: 'proxy_type', value: 'audio' }, + { key: 'duration', value: 42 }, + ]), + true, + ) + }) + + it('should populate videoStreamIndex and audioStreamIndex in mediaInfo.metadata', async () => { + const asset = await prisma.asset.create({ + data: { name: 'dual-stream.mp4', type: 'file', status: 'uploaded' }, + }) + + vi.mocked(transcodeService.getVideoInfo).mockResolvedValueOnce({ + originalWidth: 1920, + originalHeight: 1080, + duration: 15, + bitRate: 2000000, + frameRate: 30, + totalFrames: 450, + hasAudio: true, + videoCodec: 'h264', + audioCodec: 'aac', + videoStreamIndex: 1, + audioStreamIndex: 2, + mimeType: 'video/mp4', + }) + + const result = await getMediaInfoActivity({ + assetId: asset.id, + filePath: '/tmp/dual-stream.mp4', + proxyType: 'video', + mediaType: 'video/mp4', + }) + + expect(result.metadata?.videoStreamIndex).toBe(1) + expect(result.metadata?.audioStreamIndex).toBe(2) + }) + it('should call getImageInfo and set file_type to image', async () => { const asset = await prisma.asset.create({ data: { name: 'i.png', type: 'file', status: 'uploaded' }, @@ -703,6 +783,28 @@ describe('Transcode Activities', () => { ) }) + it('transcodeVideoActivity forwards streamIndex and audioStreamIndex to transcodeService', async () => { + vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found')) + + await transcodeVideoActivity({ + assetKey: 'v.mp4', + filePath: '/tmp/v.mp4', + videoSpec: { resolution: '720p', width: 1280, height: 720 }, + duration: 10, + originalFps: 30, + streamIndex: 2, + audioStreamIndex: 1, + }) + + expect(transcodeService.transcodeVideo).toHaveBeenCalledWith( + expect.objectContaining({ + inputFile: '/tmp/v.mp4', + streamIndex: 2, + audioStreamIndex: 1, + }), + ) + }) + it('transcodeVideoActivity appends -hdr.mp4 suffix and forwards HDR metadata', async () => { vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found')) @@ -946,6 +1048,44 @@ describe('Transcode Activities', () => { hdrType: 'pq', colorTransfer: 'smpte2084', }, + undefined, + ) + }) + + it('generateSpriteActivity forwards videoStreamIndex to transcodeService.generateSprite', async () => { + vi.mocked(transcodeService.generateSprite).mockImplementation( + async (_in, spriteOut, posterOut) => { + const fs = await import('fs') + fs.writeFileSync(spriteOut, 'fake-sprite') + fs.writeFileSync(posterOut, 'fake-poster') + }, + ) + + await generateSpriteActivity({ + assetKey: 'files/asset-1/video.mp4', + filePath: '/tmp/video.mp4', + mediaInfo: { + duration: 10, + metadata: { + videoStreamIndex: 1, + }, + } as unknown as PrismaJson.MediaInfo, + spriteSpec: { key: 'files/asset-1/sprite.webp', frames: 100, tileX: 10, tileY: 10 }, + posterSpec: { key: 'files/asset-1/poster.webp' }, + }) + + expect(transcodeService.generateSprite).toHaveBeenCalledWith( + '/tmp/video.mp4', + expect.any(String), + expect.any(String), + 10, + undefined, + { + isHdr: undefined, + hdrType: undefined, + colorTransfer: undefined, + }, + 1, ) }) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index 34cb0bfa..2fb44c50 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -130,54 +130,110 @@ export async function getMediaInfoActivity(params: { } if (isVideo) { - const info = await transcodeService.getVideoInfo(params.filePath) - mediaInfo.duration = info.duration - mediaInfo.frames = info.totalFrames - mediaInfo.metadata = { - originalWidth: info.originalWidth, - originalHeight: info.originalHeight, - duration: info.duration, - bitRate: info.bitRate, - videoBitRate: info.videoBitRate, - frameRate: info.frameRate, - totalFrames: info.totalFrames, - startTimecode: info.startTimecode || '00:00:00:00', - hasAudio: info.hasAudio, - videoCodec: info.videoCodec, - audioCodec: info.audioCodec, - audioChannels: info.audioChannels, - audioSampleRate: info.audioSampleRate, - audioBitDepth: info.audioBitDepth, - colorTransfer: info.colorTransfer, - colorPrimaries: info.colorPrimaries, - colorSpace: info.colorSpace, - isHdr: info.isHdr, - hdrType: info.hdrType, - dvProfile: info.dvProfile, - rotation: info.rotation, - format: {}, + let isDowngradedToAudio = false + let videoInfo: Awaited> | null = null + try { + videoInfo = await transcodeService.getVideoInfo(params.filePath) + } catch (err) { + if (err instanceof Error && err.message.includes('No video stream found')) { + try { + const audioInfo = await transcodeService.getAudioInfo(params.filePath) + isDowngradedToAudio = true + mediaInfo.proxyType = 'audio' + mediaInfo.duration = audioInfo.duration + mediaInfo.metadata = { + originalWidth: 0, + originalHeight: 0, + duration: audioInfo.duration, + bitRate: audioInfo.bitRate, + frameRate: 0, + totalFrames: 0, + startTimecode: '00:00:00:00', + hasAudio: true, + videoCodec: undefined, + audioCodec: audioInfo.audioCodec, + audioChannels: audioInfo.audioChannels, + audioSampleRate: audioInfo.audioSampleRate, + audioBitDepth: audioInfo.audioBitDepth, + audioStreamIndex: audioInfo.audioStreamIndex, + format: {}, + } + const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type') + if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio' + const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type') + if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio' + metadataUpdates.push( + { key: 'duration', value: audioInfo.duration }, + { key: 'bitRate', value: audioInfo.bitRate / 1000 }, + ) + if (audioInfo.audioCodec) + metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec }) + if (audioInfo.audioChannels !== undefined) + metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels }) + if (audioInfo.audioSampleRate !== undefined) + metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate }) + if (audioInfo.audioBitDepth !== undefined) + metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth }) + } catch { + throw err + } + } else { + throw err + } + } + + if (!isDowngradedToAudio && videoInfo) { + const info = videoInfo + mediaInfo.duration = info.duration + mediaInfo.frames = info.totalFrames + mediaInfo.metadata = { + originalWidth: info.originalWidth, + originalHeight: info.originalHeight, + duration: info.duration, + bitRate: info.bitRate, + videoBitRate: info.videoBitRate, + frameRate: info.frameRate, + totalFrames: info.totalFrames, + startTimecode: info.startTimecode || '00:00:00:00', + hasAudio: info.hasAudio, + videoCodec: info.videoCodec, + audioCodec: info.audioCodec, + audioChannels: info.audioChannels, + audioSampleRate: info.audioSampleRate, + audioBitDepth: info.audioBitDepth, + colorTransfer: info.colorTransfer, + colorPrimaries: info.colorPrimaries, + colorSpace: info.colorSpace, + isHdr: info.isHdr, + hdrType: info.hdrType, + dvProfile: info.dvProfile, + rotation: info.rotation, + videoStreamIndex: info.videoStreamIndex, + audioStreamIndex: info.audioStreamIndex, + format: {}, + } + metadataUpdates.push( + { key: 'resolution_width', value: info.originalWidth }, + { key: 'resolution_height', value: info.originalHeight }, + { key: 'duration', value: info.duration }, + { key: 'bitRate', value: info.bitRate / 1000 }, + { key: 'frame_rate', value: info.frameRate }, + ) + if (info.rotation !== undefined) + metadataUpdates.push({ key: 'rotation', value: info.rotation }) + if (info.isHdr !== undefined) + metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 }) + if (info.colorTransfer) + metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer }) + if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec }) + if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec }) + if (info.audioChannels !== undefined) + metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels }) + if (info.audioSampleRate !== undefined) + metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate }) + if (info.audioBitDepth !== undefined) + metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth }) } - metadataUpdates.push( - { key: 'resolution_width', value: info.originalWidth }, - { key: 'resolution_height', value: info.originalHeight }, - { key: 'duration', value: info.duration }, - { key: 'bitRate', value: info.bitRate / 1000 }, - { key: 'frame_rate', value: info.frameRate }, - ) - if (info.rotation !== undefined) - metadataUpdates.push({ key: 'rotation', value: info.rotation }) - if (info.isHdr !== undefined) - metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 }) - if (info.colorTransfer) - metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer }) - if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec }) - if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec }) - if (info.audioChannels !== undefined) - metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels }) - if (info.audioSampleRate !== undefined) - metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate }) - if (info.audioBitDepth !== undefined) - metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth }) } else if (isAudio) { const info = await transcodeService.getAudioInfo(params.filePath) mediaInfo.duration = info.duration @@ -195,6 +251,7 @@ export async function getMediaInfoActivity(params: { audioChannels: info.audioChannels, audioSampleRate: info.audioSampleRate, audioBitDepth: info.audioBitDepth, + audioStreamIndex: info.audioStreamIndex, format: {}, } metadataUpdates.push( @@ -277,6 +334,8 @@ export interface VideoActivityParams { sourceColorTransfer?: string sourceColorPrimaries?: string sourceColorSpace?: string + streamIndex?: number + audioStreamIndex?: number } export async function transcodeVideoActivity( @@ -338,6 +397,8 @@ export async function transcodeVideoActivity( sourceColorTransfer: params.sourceColorTransfer, sourceColorPrimaries: params.sourceColorPrimaries, sourceColorSpace: params.sourceColorSpace, + streamIndex: params.streamIndex, + audioStreamIndex: params.audioStreamIndex, }) const stat = fs.statSync(outputFile) @@ -403,6 +464,8 @@ export interface HlsActivityParams { sourceColorTransfer?: string sourceColorPrimaries?: string sourceColorSpace?: string + streamIndex?: number + audioStreamIndex?: number } export async function transcodeHlsActivity(params: HlsActivityParams): Promise { @@ -469,6 +532,8 @@ export async function transcodeHlsActivity(params: HlsActivityParams): Promise

codec: '', } - if (spec.poster) { + const isVideo = mediaInfo.proxyType === 'video' + + if (spec.poster && isVideo) { const lastSlashIndex = key.lastIndexOf('/') const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex) const posterSpec: PrismaJson.PosterInfo = { @@ -77,6 +79,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise isHdr: mediaInfo.metadata?.isHdr, hdrType: mediaInfo.metadata?.hdrType, colorTransfer: mediaInfo.metadata?.colorTransfer, + streamIndex: mediaInfo.metadata?.videoStreamIndex, }) mediaInfo.poster = posterResult.poster await executeActivity(workerQueue, updateAssetMediaActivity, { @@ -87,7 +90,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise // Generate the sprite (and its poster fallback) before the expensive proxy transcodes so the // file list can show a poster/sprite preview while the asset is still processing. - if (spec.sprite || (spec.poster && !mediaInfo.poster)) { + if ((spec.sprite || (spec.poster && !mediaInfo.poster)) && isVideo) { const lastSlashIndex = key.lastIndexOf('/') const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex) @@ -124,7 +127,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise const metadata = mediaInfo.metadata - if (mediaInfo.proxyType === 'video' && metadata) { + if (isVideo && metadata) { const videoResolutions = getTargetVideoResolutions( spec.videoStrategy || 'best_match', metadata.originalWidth, @@ -163,6 +166,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, + streamIndex: metadata.videoStreamIndex, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoPreview = videoTranscode @@ -191,6 +196,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, + streamIndex: metadata.videoStreamIndex, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoTranscodes.push(videoTranscode) @@ -219,6 +226,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, + streamIndex: metadata.videoStreamIndex, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.isHls = true mediaInfo.hls = hlsResult @@ -233,11 +242,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise assetKey: key, filePath, threads: spec.threads, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoTranscodes.push(audioTranscode) } - if (spec.thumbnail) { + if (spec.thumbnail && isVideo) { const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, { assetKey: key, filePath,