diff --git a/CHANGELOG.md b/CHANGELOG.md index a5e37505..442345ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +* **transcode**: Fix an issue where videos or audio-only files containing attached cover art or picture streams could cause transcoding to fail during poster generation or select a single static image instead of the video + * **webui**: Fix inconsistent video resolution labels across download menus, mobile file headers, and the video player by correctly deriving resolutions from the video's short side, including portrait videos and 480p videos * **trash**: Fix an issue where the "Empty Trash" confirmation dialog would close immediately instead of showing a loading state while files were being permanently deleted diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index d6ad6f52..85f8befe 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -154,6 +154,10 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams) start.toString(), '-i', inputSource, + '-map', + '0:V:0', + '-map', + '0:a:0?', '-t', (end - start).toString(), '-c', diff --git a/packages/core/src/transcode/transcode.test.ts b/packages/core/src/transcode/transcode.test.ts index 112f8169..790f44fc 100644 --- a/packages/core/src/transcode/transcode.test.ts +++ b/packages/core/src/transcode/transcode.test.ts @@ -12,6 +12,9 @@ import { parseBitrateKbps, buildHlsMasterPlaylist, rewriteM3u8WithPresignedUrls, + compareStreams, + selectPrimaryVideoStream, + selectPrimaryAudioStream, } from './transcode' import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' @@ -712,6 +715,8 @@ describe('TranscodeService', () => { '-i', 'input.wav', '-vn', + '-map', + '0:a:0?', '-c:a', 'aac', '-b:a', @@ -746,6 +751,8 @@ describe('TranscodeService', () => { '-i', 'input.wav', '-vn', + '-map', + '0:a:0?', '-c:a', 'aac', '-b:a', @@ -782,6 +789,8 @@ describe('TranscodeService', () => { '-i', 'input.wav', '-vn', + '-map', + '0:a:0?', '-c:a', 'aac', '-b:a', @@ -2831,6 +2840,187 @@ describe('TranscodeService', () => { expect(info.rotation).toBe(180) }) + it('getVideoInfo ignores attached picture streams and selects the main video stream', async () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const mockOutput = JSON.stringify({ + format: { duration: '10.0', bit_rate: '20000000' }, + streams: [ + { + index: 0, + codec_type: 'video', + codec_name: 'mjpeg', + width: 320, + height: 240, + avg_frame_rate: '0/0', + r_frame_rate: '90000/1', + disposition: { + attached_pic: 1, + }, + }, + { + index: 1, + codec_type: 'video', + codec_name: 'h264', + width: 1920, + height: 1080, + avg_frame_rate: '24/1', + r_frame_rate: '24/1', + nb_frames: '240', + disposition: { + default: 1, + attached_pic: 0, + }, + }, + ], + }) + /* eslint-enable @typescript-eslint/naming-convention */ + + mockExecFileStdout(mockOutput) + + const info = await transcodeService.getVideoInfo('cover-art-first.mp4') + expect(info.originalWidth).toBe(1920) + expect(info.originalHeight).toBe(1080) + expect(info.videoStreamIndex).toBe(1) + }) + + it('getVideoInfo throws error when file only contains audio and attached picture', async () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const mockOutput = JSON.stringify({ + format: { duration: '10.0', bit_rate: '320000' }, + streams: [ + { + index: 0, + codec_type: 'audio', + codec_name: 'aac', + channels: 2, + sample_rate: '44100', + }, + { + index: 1, + codec_type: 'video', + codec_name: 'mjpeg', + width: 320, + height: 240, + disposition: { + attached_pic: 1, + }, + }, + ], + }) + /* eslint-enable @typescript-eslint/naming-convention */ + + mockExecFileStdout(mockOutput) + + await expect(transcodeService.getVideoInfo('audio-with-cover.mp4')).rejects.toThrow( + 'No video stream found', + ) + }) + + it('compareStreams prioritizes default stream then highest bitrate', () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const s1 = { index: 0, bit_rate: '5000000', disposition: { default: 0 } } + const s2 = { index: 1, bit_rate: '2000000', disposition: { default: 1 } } + const s3 = { index: 2, bit_rate: '8000000', disposition: { default: 0 } } + const s4 = { index: 3, disposition: {} } + /* eslint-enable @typescript-eslint/naming-convention */ + + const sorted = [s1, s2, s3, s4].sort(compareStreams) + expect(sorted[0].index).toBe(1) // default: 1 + expect(sorted[1].index).toBe(2) // 8000000 bps + expect(sorted[2].index).toBe(0) // 5000000 bps + expect(sorted[3].index).toBe(3) // 0 bps + }) + + it('selectPrimaryVideoStream filters out attached picture streams', () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const streams = [ + { index: 0, codec_type: 'video', disposition: { attached_pic: 1 }, bit_rate: '9999999' }, + { index: 1, codec_type: 'audio', disposition: { default: 1 } }, + { index: 2, codec_type: 'video', disposition: { default: 0 }, bit_rate: '1500000' }, + { index: 3, codec_type: 'video', disposition: { default: 1 }, bit_rate: '1000000' }, + ] + /* eslint-enable @typescript-eslint/naming-convention */ + + const selected = selectPrimaryVideoStream(streams) + expect(selected).toBeDefined() + expect(selected?.index).toBe(3) // default: 1 non-attached + }) + + it('selectPrimaryAudioStream picks default then highest bitrate audio stream', () => { + /* eslint-disable @typescript-eslint/naming-convention */ + const streams = [ + { index: 0, codec_type: 'video', bit_rate: '2000000' }, + { index: 1, codec_type: 'audio', disposition: { default: 0 }, bit_rate: '320000' }, + { index: 2, codec_type: 'audio', disposition: { default: 1 }, bit_rate: '128000' }, + ] + /* eslint-enable @typescript-eslint/naming-convention */ + + const selected = selectPrimaryAudioStream(streams) + expect(selected).toBeDefined() + expect(selected?.index).toBe(2) // default: 1 audio + }) + + it('transcodeVideo uses explicit streamIndex and audioStreamIndex in mapping', async () => { + let executedArgs: string[] = [] + ;( + child_process.execFile as unknown as { + mockImplementation: ( + fn: ( + file: string, + args: string[], + callback: (error: Error | null, result: { stdout: string; stderr: string }) => void, + ) => void, + ) => void + } + ).mockImplementation((_file, args, callback) => { + executedArgs = args + callback(null, { stdout: '', stderr: '' }) + }) + + await transcodeService.transcodeVideo({ + inputFile: 'input.mp4', + outputFile: 'output.mp4', + width: 1280, + height: 720, + hardwareAcceleration: 'off', + streamIndex: 2, + audioStreamIndex: 1, + }) + + const filterIdx = executedArgs.indexOf('-filter_complex') + expect(filterIdx).toBeGreaterThan(-1) + expect(executedArgs[filterIdx + 1]).toContain('[0:2]scale=') + + const mapIndices = executedArgs + .map((arg, idx) => (arg === '-map' ? executedArgs[idx + 1] : null)) + .filter(Boolean) + expect(mapIndices).toContain('0:1') + }) + + it('generatePoster uses explicit streamIndex in mapping', async () => { + let executedArgs: string[] = [] + ;( + child_process.execFile as unknown as { + mockImplementation: ( + fn: ( + file: string, + args: string[], + callback: (error: Error | null, result: { stdout: string; stderr: string }) => void, + ) => void, + ) => void + } + ).mockImplementation((_file, args, callback) => { + executedArgs = args + callback(null, { stdout: '', stderr: '' }) + }) + + await transcodeService.generatePoster('input.mp4', 'poster.webp', { streamIndex: 3 }) + + const mapIdx = executedArgs.indexOf('-map') + expect(mapIdx).toBeGreaterThan(-1) + expect(executedArgs[mapIdx + 1]).toBe('0:3') + }) + it('transcodeVideo preserves HDR parameters and tags when hdr is true', async () => { let executedArgs: string[] = [] ;( @@ -3018,7 +3208,7 @@ describe('TranscodeService', () => { const filterIdx = spriteCmd!.indexOf('-filter_complex') expect(filterIdx).toBeGreaterThan(-1) const filterStr = spriteCmd![filterIdx + 1] - expect(filterStr).toContain('[0:v]fps=') + expect(filterStr).toContain('[0:V]fps=') expect(filterStr).toContain('scale=w=300:h=-2,tile=10x10[sprite_out]') }) diff --git a/packages/core/src/transcode/transcode.ts b/packages/core/src/transcode/transcode.ts index 7321ca1e..016dfc40 100644 --- a/packages/core/src/transcode/transcode.ts +++ b/packages/core/src/transcode/transcode.ts @@ -97,6 +97,54 @@ export interface GeneratePosterOptions { hdrType?: HdrType colorTransfer?: string signal?: AbortSignal + streamIndex?: number +} + +export interface FfprobeStreamLike { + index: number + codec_type?: string + codec_name?: string + bit_rate?: string + avg_frame_rate?: string + r_frame_rate?: string + nb_frames?: string + width?: number + height?: number + channels?: number + sample_rate?: string | number + bits_per_raw_sample?: string | number + bits_per_sample?: string | number + color_transfer?: string + color_primaries?: string + color_space?: string + disposition?: Record + tags?: Record + side_data_list?: Array> + mime_codec_string?: string + [key: string]: unknown +} + +/** + * Immich-compatible stream comparator: + * 1. Streams with disposition.default: 1 come first. + * 2. Ties are broken by higher bit_rate. + */ +export function compareStreams(a: T, b: T): number { + const defDiff = (b.disposition?.default ?? 0) - (a.disposition?.default ?? 0) + if (defDiff !== 0) return defDiff + const aBitrate = parseInt(a.bit_rate || '0', 10) || 0 + const bBitrate = parseInt(b.bit_rate || '0', 10) || 0 + return bBitrate - aBitrate +} + +export function selectPrimaryVideoStream(streams: T[]): T | undefined { + return streams + .filter((s) => s.codec_type === 'video' && !s.disposition?.attached_pic) + .sort(compareStreams)[0] +} + +export function selectPrimaryAudioStream(streams: T[]): T | undefined { + return streams.filter((s) => s.codec_type === 'audio').sort(compareStreams)[0] } export interface MediaMetadata { @@ -123,6 +171,8 @@ export interface MediaMetadata { dvProfile?: number dvCompatibilityId?: number rotation?: number + videoStreamIndex?: number + audioStreamIndex?: number } export interface TranscodeVideoParams { @@ -144,6 +194,8 @@ export interface TranscodeVideoParams { sourceColorTransfer?: string sourceColorPrimaries?: string sourceColorSpace?: string + streamIndex?: number + audioStreamIndex?: number } export interface TranscodeHlsRenditionParams { @@ -166,6 +218,8 @@ export interface TranscodeHlsRenditionParams { sourceColorPrimaries?: string sourceColorSpace?: string segmentDuration?: number + streamIndex?: number + audioStreamIndex?: number } export interface EncoderConfig { @@ -475,10 +529,15 @@ export class TranscodeService { return stream.codec_name } - private safeParseInt(value: string | number | undefined): number | undefined { + private safeParseInt(value: unknown): number | undefined { if (value === undefined || value === null) return undefined - const parsed = typeof value === 'string' ? parseInt(value, 10) : value - return Number.isFinite(parsed) ? (parsed as number) : undefined + const parsed = + typeof value === 'string' + ? parseInt(value, 10) + : typeof value === 'number' + ? value + : undefined + return parsed !== undefined && Number.isFinite(parsed) ? parsed : undefined } async getVideoInfo(inputFile: string): Promise { @@ -492,10 +551,8 @@ export class TranscodeService { inputFile, ]) const info = JSON.parse(stdout) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const videoStream = info.streams.find((s: any) => s.codec_type === 'video') - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const audioStream = info.streams.find((s: any) => s.codec_type === 'audio') + const videoStream = selectPrimaryVideoStream(info.streams || []) + const audioStream = selectPrimaryAudioStream(info.streams || []) if (!videoStream) { throw new Error('No video stream found') @@ -678,6 +735,8 @@ export class TranscodeService { hdrType, dvProfile, dvCompatibilityId, + videoStreamIndex: videoStream.index, + audioStreamIndex: audioStream ? audioStream.index : undefined, } } @@ -1020,11 +1079,13 @@ export class TranscodeService { args.push('-i', params.inputFile) const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'` + const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V' + if (params.overlayFile) { args.push('-i', params.overlayFile) - filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` + filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` } else { - filterComplex = `[0:v]${baseScale}` + filterComplex = `[${vPad}]${baseScale}` } if (params.frameRate) { @@ -1063,7 +1124,8 @@ export class TranscodeService { } if (!params.disableAudio) { - args.push('-map', '0:a?') + const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?' + args.push('-map', aMap) } args.push('-c:v', encoder.name) @@ -1254,11 +1316,13 @@ export class TranscodeService { args.push('-i', params.inputFile) const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'` + const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V' + if (params.overlayFile) { args.push('-i', params.overlayFile) - filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` + filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0` } else { - filterComplex = `[0:v]${baseScale}` + filterComplex = `[${vPad}]${baseScale}` } filterComplex += `,fps=${calculatedFps}` @@ -1271,7 +1335,8 @@ export class TranscodeService { args.push('-filter_complex', filterComplex, '-map', '[vout]') if (!params.disableAudio) { - args.push('-map', '0:a?') + const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?' + args.push('-map', aMap) } args.push('-c:v', encoder.name) @@ -1624,6 +1689,8 @@ export class TranscodeService { filters.push('scale=-2:300:force_original_aspect_ratio=decrease') + const vMap = options.streamIndex !== undefined ? `0:${options.streamIndex}` : '0:V:0' + const args = [ ...(isRemote ? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5'] @@ -1631,6 +1698,8 @@ export class TranscodeService { ...(isMpegTs ? [] : ['-skip_frame', 'nointra']), '-i', inputFile, + '-map', + vMap, '-vf', filters.join(','), '-fps_mode', @@ -1660,13 +1729,14 @@ export class TranscodeService { duration: number, signal?: AbortSignal, hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string }, + streamIndex?: number, ): Promise { if (signal?.aborted) { throw new Error('Sprite generation cancelled') } // 1. Generate poster first (fast, smart frame selection) - await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal }) + await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal, streamIndex }) // 2. Generate sprite let fileSize = Infinity @@ -1679,9 +1749,23 @@ export class TranscodeService { const isSmallAndShort = fileSize <= 50 * 1024 * 1024 && duration <= 30 if (isSmallAndShort) { - await this.generateSpriteSinglePass(inputFile, outputSprite, duration, signal, hdrOptions) + await this.generateSpriteSinglePass( + inputFile, + outputSprite, + duration, + signal, + hdrOptions, + streamIndex, + ) } else { - await this.generateSpriteSeekPool(inputFile, outputSprite, duration, signal, hdrOptions) + await this.generateSpriteSeekPool( + inputFile, + outputSprite, + duration, + signal, + hdrOptions, + streamIndex, + ) } } @@ -1691,9 +1775,11 @@ export class TranscodeService { duration: number, signal?: AbortSignal, hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string }, + streamIndex?: number, ): Promise { const spriteFps = 100 / duration let filterComplex: string + const vPad = streamIndex !== undefined ? `0:${streamIndex}` : '0:V' if (hdrOptions?.isHdr) { const availableFilters = await this.getAvailableFilters() @@ -1702,9 +1788,9 @@ export class TranscodeService { colorTransfer: hdrOptions.colorTransfer, availableFilters, }) - filterComplex = `[0:v]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` + filterComplex = `[${vPad}]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` } else { - filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` + filterComplex = `[${vPad}]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]` } const args = [ @@ -1735,6 +1821,7 @@ export class TranscodeService { duration: number, signal?: AbortSignal, hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string }, + streamIndex?: number, ): Promise { if (signal?.aborted) { throw new Error('Sprite generation cancelled') @@ -1778,11 +1865,14 @@ export class TranscodeService { const ts = timestamps[idx] const framePath = path.join(tmpDir, `frame_${idx.toString().padStart(3, '0')}.webp`) + const vMap = streamIndex !== undefined ? `0:${streamIndex}` : '0:V:0' const args = [ '-ss', ts.toFixed(3), '-i', inputFile, + '-map', + vMap, '-vframes', '1', '-vf', @@ -2174,8 +2264,7 @@ export class TranscodeService { inputFile, ]) const info = JSON.parse(stdout) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const audioStream = info.streams.find((s: any) => s.codec_type === 'audio') + const audioStream = selectPrimaryAudioStream(info.streams || []) if (!audioStream) { throw new Error('No audio stream found') @@ -2199,6 +2288,7 @@ export class TranscodeService { audioBitDepth: this.safeParseInt(audioStream?.bits_per_raw_sample) ?? this.safeParseInt(audioStream?.bits_per_sample), + audioStreamIndex: audioStream.index, mimeType: '', } } @@ -2209,9 +2299,23 @@ export class TranscodeService { bitrate?: string threads?: number signal?: AbortSignal + audioStreamIndex?: number }): Promise { const bitrate = params.bitrate || '128k' - const args = ['-i', params.inputFile, '-vn', '-c:a', 'aac', '-b:a', bitrate, '-ac', '2'] + const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?' + const args = [ + '-i', + params.inputFile, + '-vn', + '-map', + aMap, + '-c:a', + 'aac', + '-b:a', + bitrate, + '-ac', + '2', + ] if (params.threads && params.threads > 0) { args.push('-threads', params.threads.toString()) } @@ -2240,6 +2344,7 @@ export class TranscodeService { const numFrames = Math.max(1, params.numFrames) const step = duration / numFrames const timestamps = Array.from({ length: numFrames }, (_, i) => i * step) + const vMap = meta.videoStreamIndex !== undefined ? `0:${meta.videoStreamIndex}` : '0:V:0' await mapConcurrent(timestamps, 10, async (t, idx) => { const outputFile = path.join(params.outputDir, `${idx + 1}.webp`) @@ -2251,6 +2356,8 @@ export class TranscodeService { t.toFixed(4), '-i', params.inputFile, + '-map', + vMap, '-vframes', '1', '-vf', @@ -2375,6 +2482,8 @@ export class TranscodeService { t.toFixed(4), '-i', inputSource, + '-map', + '0:V:0', '-vframes', '1', '-vf', diff --git a/packages/db/src/prisma-json-types.ts b/packages/db/src/prisma-json-types.ts index d667ccc6..e9d45ee8 100644 --- a/packages/db/src/prisma-json-types.ts +++ b/packages/db/src/prisma-json-types.ts @@ -170,6 +170,8 @@ declare global { isHdr?: boolean hdrType?: HdrType dvProfile?: number + videoStreamIndex?: number + audioStreamIndex?: number format: unknown } diff --git a/packages/e2e/workflow/transcode-video.test.ts b/packages/e2e/workflow/transcode-video.test.ts index bd114d65..50d36047 100644 --- a/packages/e2e/workflow/transcode-video.test.ts +++ b/packages/e2e/workflow/transcode-video.test.ts @@ -8,11 +8,56 @@ import { assetService } from '@shumai/core/src/asset/asset' import { fileURLToPath } from 'url' import * as path from 'path' import * as fs from 'fs' +import * as os from 'os' +import { execFile } from 'child_process' +import { promisify } from 'util' + +const execFileAsync = promisify(execFile) const currentDir = path.dirname(fileURLToPath(import.meta.url)) const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts') const fixturesDir = path.resolve(currentDir, '../fixtures') +async function sampleFirstFrameRgb( + buffer: Buffer, + ext: string = 'mp4', +): Promise<{ r: number; g: number; b: number }> { + const tmpFile = path.join( + os.tmpdir(), + `sample-${Date.now()}-${Math.random().toString(36).slice(2)}.${ext}`, + ) + try { + fs.writeFileSync(tmpFile, buffer) + const proc = await execFileAsync( + 'ffmpeg', + [ + '-y', + '-i', + tmpFile, + '-vframes', + '1', + '-s', + '1x1', + '-f', + 'rawvideo', + '-pix_fmt', + 'rgb24', + 'pipe:1', + ], + { encoding: 'buffer' }, + ) + return { + r: proc.stdout[0], + g: proc.stdout[1], + b: proc.stdout[2], + } + } finally { + if (fs.existsSync(tmpFile)) { + fs.unlinkSync(tmpFile) + } + } +} + describe.each(['local', 'temporal'] as const)( 'Workflow E2E - transcodeVideoWorkflow (executor: %s)', (mode) => { @@ -371,5 +416,700 @@ describe.each(['local', 'temporal'] as const)( }) expect(symlinkVariant).toContain('#EXTM3U') }, 60000) + + it('Case 1: should ignore attached picture cover art (Red) and select primary video stream (Green)', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Cover Art Video Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Cover Art Video Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case1/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case1-')) + const videoPath = path.join(tmpDir, 'case1.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=green:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:a', + '-c:v:0', + 'libx264', + '-pix_fmt:v:0', + 'yuv420p', + '-c:v:1', + 'mjpeg', + '-pix_fmt:v:1', + 'yuvj420p', + '-c:a', + 'aac', + '-disposition:v:1', + 'attached_pic', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case1/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + sprite: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + sprite: { key: string } + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('video') + expect(mediaInfo.metadata.videoStreamIndex).toBe(0) + + // Verify poster is Green (not Red attached pic) + expect(mediaInfo.poster?.key).toBeDefined() + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.g).toBeGreaterThan(100) + expect(posterColor.r).toBeLessThan(50) + expect(posterColor.b).toBeLessThan(50) + + // Verify MP4 proxy is Green + expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0) + const videoObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4') + expect(videoColor.g).toBeGreaterThan(100) + expect(videoColor.r).toBeLessThan(50) + expect(videoColor.b).toBeLessThan(50) + + // Verify sprite sheet exists + expect(mediaInfo.sprite?.key).toBeDefined() + await s3Service.headObject('shumai-e2e-test-bucket-transcode', mediaInfo.sprite.key) + }, 60000) + + it('Case 2: should select higher bitrate stream (Blue) when both streams have default flag', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Bitrate Comparison Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Bitrate Comparison Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case2/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case2-')) + const videoPath = path.join(tmpDir, 'case2.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:a', + '-c:v:0', + 'libx264', + '-b:v:0', + '200k', + '-minrate:v:0', + '200k', + '-maxrate:v:0', + '200k', + '-bufsize:v:0', + '200k', + '-nal-hrd', + 'cbr', + '-disposition:v:0', + 'default', + '-c:v:1', + 'libx264', + '-b:v:1', + '2000k', + '-minrate:v:1', + '2000k', + '-maxrate:v:1', + '2000k', + '-bufsize:v:1', + '2000k', + '-nal-hrd', + 'cbr', + '-disposition:v:1', + 'default', + '-c:a', + 'aac', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case2/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('video') + // Stream 1 (Blue, 2000k) should be selected over Stream 0 (Red, 200k) + expect(mediaInfo.metadata.videoStreamIndex).toBe(1) + + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.b).toBeGreaterThan(100) + expect(posterColor.r).toBeLessThan(50) + expect(posterColor.g).toBeLessThan(50) + + const videoObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4') + expect(videoColor.b).toBeGreaterThan(100) + expect(videoColor.r).toBeLessThan(50) + expect(videoColor.g).toBeLessThan(50) + }, 60000) + + it('Case 3: should select stream with default flag (Yellow) over higher bitrate stream without default flag (Blue)', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Default Flag Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Default Flag Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case3/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case3-')) + const videoPath = path.join(tmpDir, 'case3.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=yellow:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:a', + '-c:v:0', + 'libx264', + '-b:v:0', + '200k', + '-minrate:v:0', + '200k', + '-maxrate:v:0', + '200k', + '-bufsize:v:0', + '200k', + '-nal-hrd', + 'cbr', + '-disposition:v:0', + 'default', + '-c:v:1', + 'libx264', + '-b:v:1', + '2000k', + '-minrate:v:1', + '2000k', + '-maxrate:v:1', + '2000k', + '-bufsize:v:1', + '2000k', + '-nal-hrd', + 'cbr', + '-disposition:v:1', + '0', + '-c:a', + 'aac', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case3/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('video') + // Stream 0 (Yellow, default: 1) should win over Stream 1 (Blue, 10x higher bitrate but default: 0) + expect(mediaInfo.metadata.videoStreamIndex).toBe(0) + + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.r).toBeGreaterThan(100) + expect(posterColor.g).toBeGreaterThan(100) + expect(posterColor.b).toBeLessThan(50) + + const videoObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4') + expect(videoColor.r).toBeGreaterThan(100) + expect(videoColor.g).toBeGreaterThan(100) + expect(videoColor.b).toBeLessThan(50) + }, 60000) + + it('Case 4: should handle audio-only MP4 with cover art by downgrading to audio proxy (catches audio bug)', async () => { + const team = await prisma.team.create({ + data: { name: 'E2E Audio Cover Art Team' }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Audio Cover Art Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case4/audio-cover.mp4' }, + }) + // Uploaded as video/mp4 because of .mp4 container, even though it only has audio + cover art + const asset = await prisma.asset.create({ + data: { + name: 'audio-cover.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case4-')) + const videoPath = path.join(tmpDir, 'case4.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=magenta:s=320x240:d=2', + '-map', + '0:a', + '-map', + '1:v', + '-c:a', + 'aac', + '-c:v', + 'mjpeg', + '-pix_fmt:v:0', + 'yuvj420p', + '-disposition:v:0', + 'attached_pic', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case4/audio-cover.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + // Spec requested poster, sprite, videoStrategy as video upload would + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + poster: true, + sprite: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + // Must complete without error (previously failed trying to generate poster/sprite for audio) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + poster?: unknown + sprite?: unknown + videoTranscodes: { key: string }[] + } + expect(mediaInfo.proxyType).toBe('audio') + expect(mediaInfo.poster).toBeUndefined() + expect(mediaInfo.sprite).toBeUndefined() + expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0) + expect(mediaInfo.videoTranscodes[0].key).toContain('-audio-proxy.mp4') + await s3Service.headObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.videoTranscodes[0].key, + ) + }, 60000) + + it('Case 5: should rank 3 video streams with HLS enabled and select highest bitrate stream (Blue)', async () => { + const team = await prisma.team.create({ + data: { + name: 'E2E Multi Stream HLS Team', + settings: { + transcode: { + videoStrategy: 'best_match', + hlsEnabled: true, + hlsResolutions: ['480p'], + }, + }, + }, + }) + const project = await prisma.project.create({ + data: { name: 'E2E Multi Stream HLS Project', teamId: team.id }, + }) + const storageKey = await prisma.storageKey.create({ + data: { key: 'projects/e2e/case5/video.mp4' }, + }) + const asset = await prisma.asset.create({ + data: { + name: 'video.mp4', + type: 'file', + status: 'uploaded', + mediaType: 'video/mp4', + projectId: project.id, + storageKeyId: storageKey.id, + }, + }) + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case5-')) + const videoPath = path.join(tmpDir, 'case5.mp4') + try { + await execFileAsync('ffmpeg', [ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=green:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x240:d=2', + '-f', + 'lavfi', + '-i', + 'sine=f=1000:d=2', + '-map', + '0:v', + '-map', + '1:v', + '-map', + '2:v', + '-map', + '3:a', + '-c:v:0', + 'libx264', + '-b:v:0', + '100k', + '-minrate:v:0', + '100k', + '-maxrate:v:0', + '100k', + '-bufsize:v:0', + '100k', + '-nal-hrd', + 'cbr', + '-disposition:v:0', + 'default', + '-c:v:1', + 'libx264', + '-b:v:1', + '500k', + '-minrate:v:1', + '500k', + '-maxrate:v:1', + '500k', + '-bufsize:v:1', + '500k', + '-nal-hrd', + 'cbr', + '-disposition:v:1', + 'default', + '-c:v:2', + 'libx264', + '-b:v:2', + '1500k', + '-minrate:v:2', + '1500k', + '-maxrate:v:2', + '1500k', + '-bufsize:v:2', + '1500k', + '-nal-hrd', + 'cbr', + '-disposition:v:2', + 'default', + '-c:a', + 'aac', + videoPath, + ]) + const mp4Buffer = fs.readFileSync(videoPath) + await s3Service.putObject( + 'shumai-e2e-test-bucket-transcode', + 'projects/e2e/case5/video.mp4', + mp4Buffer, + mp4Buffer.length, + 'video/mp4', + ) + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + + const task = await prisma.workflowTask.create({ + data: { + type: 'transcode_video', + status: 'pending', + assetId: asset.id, + projectId: project.id, + teamId: team.id, + payload: { + projectId: project.id, + transcode: { + videoStrategy: 'best_match', + hlsEnabled: true, + hlsResolutions: ['480p'], + poster: true, + }, + }, + }, + }) + + const completedTask = await workflowService.executeWait(task, 45000) + expect(completedTask.status).toBe('completed') + + const updatedAsset = await prisma.asset.findUnique({ + where: { id: asset.id }, + }) + expect(updatedAsset?.status).toBe(AssetStatus.processed) + + const mediaInfo = updatedAsset?.media as unknown as { + proxyType: string + metadata: { videoStreamIndex: number } + poster: { key: string } + isHls: boolean + hls: { key: string; resolutions: { resolution: string }[] } + } + expect(mediaInfo.proxyType).toBe('video') + // Stream 2 (Blue, 1500k) should be selected + expect(mediaInfo.metadata.videoStreamIndex).toBe(2) + + const posterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.poster.key, + ) + const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp') + expect(posterColor.b).toBeGreaterThan(100) + expect(posterColor.r).toBeLessThan(50) + expect(posterColor.g).toBeLessThan(50) + + expect(mediaInfo.isHls).toBe(true) + const masterObj = await s3Service.getObject( + 'shumai-e2e-test-bucket-transcode', + mediaInfo.hls.key, + ) + expect(masterObj.buffer.toString('utf-8')).toContain('480p/index.m3u8') + }, 60000) }, ) diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 70fdd322..3c9b19d1 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -217,6 +217,86 @@ describe('Transcode Activities', () => { ) }) + it('should downgrade proxyType to audio when getVideoInfo throws No video stream found but audio exists', async () => { + const asset = await prisma.asset.create({ + data: { name: 'audio-with-art.mp4', type: 'file', status: 'uploaded' }, + }) + + vi.mocked(transcodeService.getVideoInfo).mockRejectedValueOnce( + new Error('No video stream found'), + ) + vi.mocked(transcodeService.getAudioInfo).mockResolvedValueOnce({ + originalWidth: 0, + originalHeight: 0, + duration: 42, + bitRate: 192000, + frameRate: 0, + totalFrames: 0, + hasAudio: true, + audioCodec: 'aac', + audioChannels: 2, + audioSampleRate: 44100, + audioBitDepth: 16, + audioStreamIndex: 0, + mimeType: 'audio/mp4', + }) + + const result = await getMediaInfoActivity({ + assetId: asset.id, + filePath: '/tmp/audio-with-art.mp4', + proxyType: 'video', + mediaType: 'video/mp4', + }) + + expect(transcodeService.getVideoInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4') + expect(transcodeService.getAudioInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4') + expect(result.proxyType).toBe('audio') + expect(result.duration).toBe(42) + expect(result.metadata?.hasAudio).toBe(true) + expect(result.metadata?.audioStreamIndex).toBe(0) + expect(result.metadata?.videoStreamIndex).toBeUndefined() + expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith( + asset.id, + expect.arrayContaining([ + { key: 'file_type', value: 'audio' }, + { key: 'proxy_type', value: 'audio' }, + { key: 'duration', value: 42 }, + ]), + true, + ) + }) + + it('should populate videoStreamIndex and audioStreamIndex in mediaInfo.metadata', async () => { + const asset = await prisma.asset.create({ + data: { name: 'dual-stream.mp4', type: 'file', status: 'uploaded' }, + }) + + vi.mocked(transcodeService.getVideoInfo).mockResolvedValueOnce({ + originalWidth: 1920, + originalHeight: 1080, + duration: 15, + bitRate: 2000000, + frameRate: 30, + totalFrames: 450, + hasAudio: true, + videoCodec: 'h264', + audioCodec: 'aac', + videoStreamIndex: 1, + audioStreamIndex: 2, + mimeType: 'video/mp4', + }) + + const result = await getMediaInfoActivity({ + assetId: asset.id, + filePath: '/tmp/dual-stream.mp4', + proxyType: 'video', + mediaType: 'video/mp4', + }) + + expect(result.metadata?.videoStreamIndex).toBe(1) + expect(result.metadata?.audioStreamIndex).toBe(2) + }) + it('should call getImageInfo and set file_type to image', async () => { const asset = await prisma.asset.create({ data: { name: 'i.png', type: 'file', status: 'uploaded' }, @@ -703,6 +783,28 @@ describe('Transcode Activities', () => { ) }) + it('transcodeVideoActivity forwards streamIndex and audioStreamIndex to transcodeService', async () => { + vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found')) + + await transcodeVideoActivity({ + assetKey: 'v.mp4', + filePath: '/tmp/v.mp4', + videoSpec: { resolution: '720p', width: 1280, height: 720 }, + duration: 10, + originalFps: 30, + streamIndex: 2, + audioStreamIndex: 1, + }) + + expect(transcodeService.transcodeVideo).toHaveBeenCalledWith( + expect.objectContaining({ + inputFile: '/tmp/v.mp4', + streamIndex: 2, + audioStreamIndex: 1, + }), + ) + }) + it('transcodeVideoActivity appends -hdr.mp4 suffix and forwards HDR metadata', async () => { vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found')) @@ -946,6 +1048,44 @@ describe('Transcode Activities', () => { hdrType: 'pq', colorTransfer: 'smpte2084', }, + undefined, + ) + }) + + it('generateSpriteActivity forwards videoStreamIndex to transcodeService.generateSprite', async () => { + vi.mocked(transcodeService.generateSprite).mockImplementation( + async (_in, spriteOut, posterOut) => { + const fs = await import('fs') + fs.writeFileSync(spriteOut, 'fake-sprite') + fs.writeFileSync(posterOut, 'fake-poster') + }, + ) + + await generateSpriteActivity({ + assetKey: 'files/asset-1/video.mp4', + filePath: '/tmp/video.mp4', + mediaInfo: { + duration: 10, + metadata: { + videoStreamIndex: 1, + }, + } as unknown as PrismaJson.MediaInfo, + spriteSpec: { key: 'files/asset-1/sprite.webp', frames: 100, tileX: 10, tileY: 10 }, + posterSpec: { key: 'files/asset-1/poster.webp' }, + }) + + expect(transcodeService.generateSprite).toHaveBeenCalledWith( + '/tmp/video.mp4', + expect.any(String), + expect.any(String), + 10, + undefined, + { + isHdr: undefined, + hdrType: undefined, + colorTransfer: undefined, + }, + 1, ) }) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index 34cb0bfa..2fb44c50 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -130,54 +130,110 @@ export async function getMediaInfoActivity(params: { } if (isVideo) { - const info = await transcodeService.getVideoInfo(params.filePath) - mediaInfo.duration = info.duration - mediaInfo.frames = info.totalFrames - mediaInfo.metadata = { - originalWidth: info.originalWidth, - originalHeight: info.originalHeight, - duration: info.duration, - bitRate: info.bitRate, - videoBitRate: info.videoBitRate, - frameRate: info.frameRate, - totalFrames: info.totalFrames, - startTimecode: info.startTimecode || '00:00:00:00', - hasAudio: info.hasAudio, - videoCodec: info.videoCodec, - audioCodec: info.audioCodec, - audioChannels: info.audioChannels, - audioSampleRate: info.audioSampleRate, - audioBitDepth: info.audioBitDepth, - colorTransfer: info.colorTransfer, - colorPrimaries: info.colorPrimaries, - colorSpace: info.colorSpace, - isHdr: info.isHdr, - hdrType: info.hdrType, - dvProfile: info.dvProfile, - rotation: info.rotation, - format: {}, + let isDowngradedToAudio = false + let videoInfo: Awaited> | null = null + try { + videoInfo = await transcodeService.getVideoInfo(params.filePath) + } catch (err) { + if (err instanceof Error && err.message.includes('No video stream found')) { + try { + const audioInfo = await transcodeService.getAudioInfo(params.filePath) + isDowngradedToAudio = true + mediaInfo.proxyType = 'audio' + mediaInfo.duration = audioInfo.duration + mediaInfo.metadata = { + originalWidth: 0, + originalHeight: 0, + duration: audioInfo.duration, + bitRate: audioInfo.bitRate, + frameRate: 0, + totalFrames: 0, + startTimecode: '00:00:00:00', + hasAudio: true, + videoCodec: undefined, + audioCodec: audioInfo.audioCodec, + audioChannels: audioInfo.audioChannels, + audioSampleRate: audioInfo.audioSampleRate, + audioBitDepth: audioInfo.audioBitDepth, + audioStreamIndex: audioInfo.audioStreamIndex, + format: {}, + } + const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type') + if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio' + const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type') + if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio' + metadataUpdates.push( + { key: 'duration', value: audioInfo.duration }, + { key: 'bitRate', value: audioInfo.bitRate / 1000 }, + ) + if (audioInfo.audioCodec) + metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec }) + if (audioInfo.audioChannels !== undefined) + metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels }) + if (audioInfo.audioSampleRate !== undefined) + metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate }) + if (audioInfo.audioBitDepth !== undefined) + metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth }) + } catch { + throw err + } + } else { + throw err + } + } + + if (!isDowngradedToAudio && videoInfo) { + const info = videoInfo + mediaInfo.duration = info.duration + mediaInfo.frames = info.totalFrames + mediaInfo.metadata = { + originalWidth: info.originalWidth, + originalHeight: info.originalHeight, + duration: info.duration, + bitRate: info.bitRate, + videoBitRate: info.videoBitRate, + frameRate: info.frameRate, + totalFrames: info.totalFrames, + startTimecode: info.startTimecode || '00:00:00:00', + hasAudio: info.hasAudio, + videoCodec: info.videoCodec, + audioCodec: info.audioCodec, + audioChannels: info.audioChannels, + audioSampleRate: info.audioSampleRate, + audioBitDepth: info.audioBitDepth, + colorTransfer: info.colorTransfer, + colorPrimaries: info.colorPrimaries, + colorSpace: info.colorSpace, + isHdr: info.isHdr, + hdrType: info.hdrType, + dvProfile: info.dvProfile, + rotation: info.rotation, + videoStreamIndex: info.videoStreamIndex, + audioStreamIndex: info.audioStreamIndex, + format: {}, + } + metadataUpdates.push( + { key: 'resolution_width', value: info.originalWidth }, + { key: 'resolution_height', value: info.originalHeight }, + { key: 'duration', value: info.duration }, + { key: 'bitRate', value: info.bitRate / 1000 }, + { key: 'frame_rate', value: info.frameRate }, + ) + if (info.rotation !== undefined) + metadataUpdates.push({ key: 'rotation', value: info.rotation }) + if (info.isHdr !== undefined) + metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 }) + if (info.colorTransfer) + metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer }) + if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec }) + if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec }) + if (info.audioChannels !== undefined) + metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels }) + if (info.audioSampleRate !== undefined) + metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate }) + if (info.audioBitDepth !== undefined) + metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth }) } - metadataUpdates.push( - { key: 'resolution_width', value: info.originalWidth }, - { key: 'resolution_height', value: info.originalHeight }, - { key: 'duration', value: info.duration }, - { key: 'bitRate', value: info.bitRate / 1000 }, - { key: 'frame_rate', value: info.frameRate }, - ) - if (info.rotation !== undefined) - metadataUpdates.push({ key: 'rotation', value: info.rotation }) - if (info.isHdr !== undefined) - metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 }) - if (info.colorTransfer) - metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer }) - if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec }) - if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec }) - if (info.audioChannels !== undefined) - metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels }) - if (info.audioSampleRate !== undefined) - metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate }) - if (info.audioBitDepth !== undefined) - metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth }) } else if (isAudio) { const info = await transcodeService.getAudioInfo(params.filePath) mediaInfo.duration = info.duration @@ -195,6 +251,7 @@ export async function getMediaInfoActivity(params: { audioChannels: info.audioChannels, audioSampleRate: info.audioSampleRate, audioBitDepth: info.audioBitDepth, + audioStreamIndex: info.audioStreamIndex, format: {}, } metadataUpdates.push( @@ -277,6 +334,8 @@ export interface VideoActivityParams { sourceColorTransfer?: string sourceColorPrimaries?: string sourceColorSpace?: string + streamIndex?: number + audioStreamIndex?: number } export async function transcodeVideoActivity( @@ -338,6 +397,8 @@ export async function transcodeVideoActivity( sourceColorTransfer: params.sourceColorTransfer, sourceColorPrimaries: params.sourceColorPrimaries, sourceColorSpace: params.sourceColorSpace, + streamIndex: params.streamIndex, + audioStreamIndex: params.audioStreamIndex, }) const stat = fs.statSync(outputFile) @@ -403,6 +464,8 @@ export interface HlsActivityParams { sourceColorTransfer?: string sourceColorPrimaries?: string sourceColorSpace?: string + streamIndex?: number + audioStreamIndex?: number } export async function transcodeHlsActivity(params: HlsActivityParams): Promise { @@ -469,6 +532,8 @@ export async function transcodeHlsActivity(params: HlsActivityParams): Promise

codec: '', } - if (spec.poster) { + const isVideo = mediaInfo.proxyType === 'video' + + if (spec.poster && isVideo) { const lastSlashIndex = key.lastIndexOf('/') const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex) const posterSpec: PrismaJson.PosterInfo = { @@ -77,6 +79,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise isHdr: mediaInfo.metadata?.isHdr, hdrType: mediaInfo.metadata?.hdrType, colorTransfer: mediaInfo.metadata?.colorTransfer, + streamIndex: mediaInfo.metadata?.videoStreamIndex, }) mediaInfo.poster = posterResult.poster await executeActivity(workerQueue, updateAssetMediaActivity, { @@ -87,7 +90,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise // Generate the sprite (and its poster fallback) before the expensive proxy transcodes so the // file list can show a poster/sprite preview while the asset is still processing. - if (spec.sprite || (spec.poster && !mediaInfo.poster)) { + if ((spec.sprite || (spec.poster && !mediaInfo.poster)) && isVideo) { const lastSlashIndex = key.lastIndexOf('/') const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex) @@ -124,7 +127,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise const metadata = mediaInfo.metadata - if (mediaInfo.proxyType === 'video' && metadata) { + if (isVideo && metadata) { const videoResolutions = getTargetVideoResolutions( spec.videoStrategy || 'best_match', metadata.originalWidth, @@ -163,6 +166,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, + streamIndex: metadata.videoStreamIndex, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoPreview = videoTranscode @@ -191,6 +196,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, + streamIndex: metadata.videoStreamIndex, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoTranscodes.push(videoTranscode) @@ -219,6 +226,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, + streamIndex: metadata.videoStreamIndex, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.isHls = true mediaInfo.hls = hlsResult @@ -233,11 +242,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise assetKey: key, filePath, threads: spec.threads, + audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoTranscodes.push(audioTranscode) } - if (spec.thumbnail) { + if (spec.thumbnail && isVideo) { const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, { assetKey: key, filePath,