fix(transcode): avoid selecting attached picture streams for video transcoding (#443)

* fix(transcode): avoid selecting attached picture streams for video transcoding

Filter out attached picture streams (disposition.attached_pic) during video probing and stream selection, matching Immich's default and bitrate selection strategy. Forward explicit primary video and audio stream indices to FFmpeg pipelines to prevent picking embedded cover art or thumbnail streams.

* fix(transcode): skip poster and sprite generation for audio-only assets and add multi-stream e2e tests

* test(workflow): scope pixel format to h264 stream and use yuvj420p for mjpeg cover art
This commit is contained in:
Yiling-J
2026-10-02 13:51:17 +08:00
committed by GitHub
parent 206763e966
commit 3564d4d164
10 changed files with 1346 additions and 73 deletions
+2
View File
@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Fixed
* **transcode**: Fix an issue where videos or audio-only files containing attached cover art or picture streams could cause transcoding to fail during poster generation or select a single static image instead of the video
* **webui**: Fix inconsistent video resolution labels across download menus, mobile file headers, and the video player by correctly deriving resolutions from the video's short side, including portrait videos and 480p videos
* **trash**: Fix an issue where the "Empty Trash" confirmation dialog would close immediately instead of showing a loading state while files were being permanently deleted
+4
View File
@@ -154,6 +154,10 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
start.toString(),
'-i',
inputSource,
'-map',
'0:V:0',
'-map',
'0:a:0?',
'-t',
(end - start).toString(),
'-c',
+191 -1
View File
@@ -12,6 +12,9 @@ import {
parseBitrateKbps,
buildHlsMasterPlaylist,
rewriteM3u8WithPresignedUrls,
compareStreams,
selectPrimaryVideoStream,
selectPrimaryAudioStream,
} from './transcode'
import { logger } from '@shumai/core/src/logger'
import { s3Service } from '@shumai/core/src/s3/s3'
@@ -712,6 +715,8 @@ describe('TranscodeService', () => {
'-i',
'input.wav',
'-vn',
'-map',
'0:a:0?',
'-c:a',
'aac',
'-b:a',
@@ -746,6 +751,8 @@ describe('TranscodeService', () => {
'-i',
'input.wav',
'-vn',
'-map',
'0:a:0?',
'-c:a',
'aac',
'-b:a',
@@ -782,6 +789,8 @@ describe('TranscodeService', () => {
'-i',
'input.wav',
'-vn',
'-map',
'0:a:0?',
'-c:a',
'aac',
'-b:a',
@@ -2831,6 +2840,187 @@ describe('TranscodeService', () => {
expect(info.rotation).toBe(180)
})
it('getVideoInfo ignores attached picture streams and selects the main video stream', async () => {
/* eslint-disable @typescript-eslint/naming-convention */
const mockOutput = JSON.stringify({
format: { duration: '10.0', bit_rate: '20000000' },
streams: [
{
index: 0,
codec_type: 'video',
codec_name: 'mjpeg',
width: 320,
height: 240,
avg_frame_rate: '0/0',
r_frame_rate: '90000/1',
disposition: {
attached_pic: 1,
},
},
{
index: 1,
codec_type: 'video',
codec_name: 'h264',
width: 1920,
height: 1080,
avg_frame_rate: '24/1',
r_frame_rate: '24/1',
nb_frames: '240',
disposition: {
default: 1,
attached_pic: 0,
},
},
],
})
/* eslint-enable @typescript-eslint/naming-convention */
mockExecFileStdout(mockOutput)
const info = await transcodeService.getVideoInfo('cover-art-first.mp4')
expect(info.originalWidth).toBe(1920)
expect(info.originalHeight).toBe(1080)
expect(info.videoStreamIndex).toBe(1)
})
it('getVideoInfo throws error when file only contains audio and attached picture', async () => {
/* eslint-disable @typescript-eslint/naming-convention */
const mockOutput = JSON.stringify({
format: { duration: '10.0', bit_rate: '320000' },
streams: [
{
index: 0,
codec_type: 'audio',
codec_name: 'aac',
channels: 2,
sample_rate: '44100',
},
{
index: 1,
codec_type: 'video',
codec_name: 'mjpeg',
width: 320,
height: 240,
disposition: {
attached_pic: 1,
},
},
],
})
/* eslint-enable @typescript-eslint/naming-convention */
mockExecFileStdout(mockOutput)
await expect(transcodeService.getVideoInfo('audio-with-cover.mp4')).rejects.toThrow(
'No video stream found',
)
})
it('compareStreams prioritizes default stream then highest bitrate', () => {
/* eslint-disable @typescript-eslint/naming-convention */
const s1 = { index: 0, bit_rate: '5000000', disposition: { default: 0 } }
const s2 = { index: 1, bit_rate: '2000000', disposition: { default: 1 } }
const s3 = { index: 2, bit_rate: '8000000', disposition: { default: 0 } }
const s4 = { index: 3, disposition: {} }
/* eslint-enable @typescript-eslint/naming-convention */
const sorted = [s1, s2, s3, s4].sort(compareStreams)
expect(sorted[0].index).toBe(1) // default: 1
expect(sorted[1].index).toBe(2) // 8000000 bps
expect(sorted[2].index).toBe(0) // 5000000 bps
expect(sorted[3].index).toBe(3) // 0 bps
})
it('selectPrimaryVideoStream filters out attached picture streams', () => {
/* eslint-disable @typescript-eslint/naming-convention */
const streams = [
{ index: 0, codec_type: 'video', disposition: { attached_pic: 1 }, bit_rate: '9999999' },
{ index: 1, codec_type: 'audio', disposition: { default: 1 } },
{ index: 2, codec_type: 'video', disposition: { default: 0 }, bit_rate: '1500000' },
{ index: 3, codec_type: 'video', disposition: { default: 1 }, bit_rate: '1000000' },
]
/* eslint-enable @typescript-eslint/naming-convention */
const selected = selectPrimaryVideoStream(streams)
expect(selected).toBeDefined()
expect(selected?.index).toBe(3) // default: 1 non-attached
})
it('selectPrimaryAudioStream picks default then highest bitrate audio stream', () => {
/* eslint-disable @typescript-eslint/naming-convention */
const streams = [
{ index: 0, codec_type: 'video', bit_rate: '2000000' },
{ index: 1, codec_type: 'audio', disposition: { default: 0 }, bit_rate: '320000' },
{ index: 2, codec_type: 'audio', disposition: { default: 1 }, bit_rate: '128000' },
]
/* eslint-enable @typescript-eslint/naming-convention */
const selected = selectPrimaryAudioStream(streams)
expect(selected).toBeDefined()
expect(selected?.index).toBe(2) // default: 1 audio
})
it('transcodeVideo uses explicit streamIndex and audioStreamIndex in mapping', async () => {
let executedArgs: string[] = []
;(
child_process.execFile as unknown as {
mockImplementation: (
fn: (
file: string,
args: string[],
callback: (error: Error | null, result: { stdout: string; stderr: string }) => void,
) => void,
) => void
}
).mockImplementation((_file, args, callback) => {
executedArgs = args
callback(null, { stdout: '', stderr: '' })
})
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile: 'output.mp4',
width: 1280,
height: 720,
hardwareAcceleration: 'off',
streamIndex: 2,
audioStreamIndex: 1,
})
const filterIdx = executedArgs.indexOf('-filter_complex')
expect(filterIdx).toBeGreaterThan(-1)
expect(executedArgs[filterIdx + 1]).toContain('[0:2]scale=')
const mapIndices = executedArgs
.map((arg, idx) => (arg === '-map' ? executedArgs[idx + 1] : null))
.filter(Boolean)
expect(mapIndices).toContain('0:1')
})
it('generatePoster uses explicit streamIndex in mapping', async () => {
let executedArgs: string[] = []
;(
child_process.execFile as unknown as {
mockImplementation: (
fn: (
file: string,
args: string[],
callback: (error: Error | null, result: { stdout: string; stderr: string }) => void,
) => void,
) => void
}
).mockImplementation((_file, args, callback) => {
executedArgs = args
callback(null, { stdout: '', stderr: '' })
})
await transcodeService.generatePoster('input.mp4', 'poster.webp', { streamIndex: 3 })
const mapIdx = executedArgs.indexOf('-map')
expect(mapIdx).toBeGreaterThan(-1)
expect(executedArgs[mapIdx + 1]).toBe('0:3')
})
it('transcodeVideo preserves HDR parameters and tags when hdr is true', async () => {
let executedArgs: string[] = []
;(
@@ -3018,7 +3208,7 @@ describe('TranscodeService', () => {
const filterIdx = spriteCmd!.indexOf('-filter_complex')
expect(filterIdx).toBeGreaterThan(-1)
const filterStr = spriteCmd![filterIdx + 1]
expect(filterStr).toContain('[0:v]fps=')
expect(filterStr).toContain('[0:V]fps=')
expect(filterStr).toContain('scale=w=300:h=-2,tile=10x10[sprite_out]')
})
+130 -21
View File
@@ -97,6 +97,54 @@ export interface GeneratePosterOptions {
hdrType?: HdrType
colorTransfer?: string
signal?: AbortSignal
streamIndex?: number
}
export interface FfprobeStreamLike {
index: number
codec_type?: string
codec_name?: string
bit_rate?: string
avg_frame_rate?: string
r_frame_rate?: string
nb_frames?: string
width?: number
height?: number
channels?: number
sample_rate?: string | number
bits_per_raw_sample?: string | number
bits_per_sample?: string | number
color_transfer?: string
color_primaries?: string
color_space?: string
disposition?: Record<string, number | undefined>
tags?: Record<string, string | undefined>
side_data_list?: Array<Record<string, unknown>>
mime_codec_string?: string
[key: string]: unknown
}
/**
* Immich-compatible stream comparator:
* 1. Streams with disposition.default: 1 come first.
* 2. Ties are broken by higher bit_rate.
*/
export function compareStreams<T extends FfprobeStreamLike>(a: T, b: T): number {
const defDiff = (b.disposition?.default ?? 0) - (a.disposition?.default ?? 0)
if (defDiff !== 0) return defDiff
const aBitrate = parseInt(a.bit_rate || '0', 10) || 0
const bBitrate = parseInt(b.bit_rate || '0', 10) || 0
return bBitrate - aBitrate
}
export function selectPrimaryVideoStream<T extends FfprobeStreamLike>(streams: T[]): T | undefined {
return streams
.filter((s) => s.codec_type === 'video' && !s.disposition?.attached_pic)
.sort(compareStreams)[0]
}
export function selectPrimaryAudioStream<T extends FfprobeStreamLike>(streams: T[]): T | undefined {
return streams.filter((s) => s.codec_type === 'audio').sort(compareStreams)[0]
}
export interface MediaMetadata {
@@ -123,6 +171,8 @@ export interface MediaMetadata {
dvProfile?: number
dvCompatibilityId?: number
rotation?: number
videoStreamIndex?: number
audioStreamIndex?: number
}
export interface TranscodeVideoParams {
@@ -144,6 +194,8 @@ export interface TranscodeVideoParams {
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
streamIndex?: number
audioStreamIndex?: number
}
export interface TranscodeHlsRenditionParams {
@@ -166,6 +218,8 @@ export interface TranscodeHlsRenditionParams {
sourceColorPrimaries?: string
sourceColorSpace?: string
segmentDuration?: number
streamIndex?: number
audioStreamIndex?: number
}
export interface EncoderConfig {
@@ -475,10 +529,15 @@ export class TranscodeService {
return stream.codec_name
}
private safeParseInt(value: string | number | undefined): number | undefined {
private safeParseInt(value: unknown): number | undefined {
if (value === undefined || value === null) return undefined
const parsed = typeof value === 'string' ? parseInt(value, 10) : value
return Number.isFinite(parsed) ? (parsed as number) : undefined
const parsed =
typeof value === 'string'
? parseInt(value, 10)
: typeof value === 'number'
? value
: undefined
return parsed !== undefined && Number.isFinite(parsed) ? parsed : undefined
}
async getVideoInfo(inputFile: string): Promise<MediaMetadata> {
@@ -492,10 +551,8 @@ export class TranscodeService {
inputFile,
])
const info = JSON.parse(stdout)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const videoStream = info.streams.find((s: any) => s.codec_type === 'video')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const audioStream = info.streams.find((s: any) => s.codec_type === 'audio')
const videoStream = selectPrimaryVideoStream(info.streams || [])
const audioStream = selectPrimaryAudioStream(info.streams || [])
if (!videoStream) {
throw new Error('No video stream found')
@@ -678,6 +735,8 @@ export class TranscodeService {
hdrType,
dvProfile,
dvCompatibilityId,
videoStreamIndex: videoStream.index,
audioStreamIndex: audioStream ? audioStream.index : undefined,
}
}
@@ -1020,11 +1079,13 @@ export class TranscodeService {
args.push('-i', params.inputFile)
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
if (params.overlayFile) {
args.push('-i', params.overlayFile)
filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
} else {
filterComplex = `[0:v]${baseScale}`
filterComplex = `[${vPad}]${baseScale}`
}
if (params.frameRate) {
@@ -1063,7 +1124,8 @@ export class TranscodeService {
}
if (!params.disableAudio) {
args.push('-map', '0:a?')
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
args.push('-map', aMap)
}
args.push('-c:v', encoder.name)
@@ -1254,11 +1316,13 @@ export class TranscodeService {
args.push('-i', params.inputFile)
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
if (params.overlayFile) {
args.push('-i', params.overlayFile)
filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
} else {
filterComplex = `[0:v]${baseScale}`
filterComplex = `[${vPad}]${baseScale}`
}
filterComplex += `,fps=${calculatedFps}`
@@ -1271,7 +1335,8 @@ export class TranscodeService {
args.push('-filter_complex', filterComplex, '-map', '[vout]')
if (!params.disableAudio) {
args.push('-map', '0:a?')
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
args.push('-map', aMap)
}
args.push('-c:v', encoder.name)
@@ -1624,6 +1689,8 @@ export class TranscodeService {
filters.push('scale=-2:300:force_original_aspect_ratio=decrease')
const vMap = options.streamIndex !== undefined ? `0:${options.streamIndex}` : '0:V:0'
const args = [
...(isRemote
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
@@ -1631,6 +1698,8 @@ export class TranscodeService {
...(isMpegTs ? [] : ['-skip_frame', 'nointra']),
'-i',
inputFile,
'-map',
vMap,
'-vf',
filters.join(','),
'-fps_mode',
@@ -1660,13 +1729,14 @@ export class TranscodeService {
duration: number,
signal?: AbortSignal,
hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string },
streamIndex?: number,
): Promise<void> {
if (signal?.aborted) {
throw new Error('Sprite generation cancelled')
}
// 1. Generate poster first (fast, smart frame selection)
await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal })
await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal, streamIndex })
// 2. Generate sprite
let fileSize = Infinity
@@ -1679,9 +1749,23 @@ export class TranscodeService {
const isSmallAndShort = fileSize <= 50 * 1024 * 1024 && duration <= 30
if (isSmallAndShort) {
await this.generateSpriteSinglePass(inputFile, outputSprite, duration, signal, hdrOptions)
await this.generateSpriteSinglePass(
inputFile,
outputSprite,
duration,
signal,
hdrOptions,
streamIndex,
)
} else {
await this.generateSpriteSeekPool(inputFile, outputSprite, duration, signal, hdrOptions)
await this.generateSpriteSeekPool(
inputFile,
outputSprite,
duration,
signal,
hdrOptions,
streamIndex,
)
}
}
@@ -1691,9 +1775,11 @@ export class TranscodeService {
duration: number,
signal?: AbortSignal,
hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string },
streamIndex?: number,
): Promise<void> {
const spriteFps = 100 / duration
let filterComplex: string
const vPad = streamIndex !== undefined ? `0:${streamIndex}` : '0:V'
if (hdrOptions?.isHdr) {
const availableFilters = await this.getAvailableFilters()
@@ -1702,9 +1788,9 @@ export class TranscodeService {
colorTransfer: hdrOptions.colorTransfer,
availableFilters,
})
filterComplex = `[0:v]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
filterComplex = `[${vPad}]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
} else {
filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
filterComplex = `[${vPad}]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
}
const args = [
@@ -1735,6 +1821,7 @@ export class TranscodeService {
duration: number,
signal?: AbortSignal,
hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string },
streamIndex?: number,
): Promise<void> {
if (signal?.aborted) {
throw new Error('Sprite generation cancelled')
@@ -1778,11 +1865,14 @@ export class TranscodeService {
const ts = timestamps[idx]
const framePath = path.join(tmpDir, `frame_${idx.toString().padStart(3, '0')}.webp`)
const vMap = streamIndex !== undefined ? `0:${streamIndex}` : '0:V:0'
const args = [
'-ss',
ts.toFixed(3),
'-i',
inputFile,
'-map',
vMap,
'-vframes',
'1',
'-vf',
@@ -2174,8 +2264,7 @@ export class TranscodeService {
inputFile,
])
const info = JSON.parse(stdout)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const audioStream = info.streams.find((s: any) => s.codec_type === 'audio')
const audioStream = selectPrimaryAudioStream(info.streams || [])
if (!audioStream) {
throw new Error('No audio stream found')
@@ -2199,6 +2288,7 @@ export class TranscodeService {
audioBitDepth:
this.safeParseInt(audioStream?.bits_per_raw_sample) ??
this.safeParseInt(audioStream?.bits_per_sample),
audioStreamIndex: audioStream.index,
mimeType: '',
}
}
@@ -2209,9 +2299,23 @@ export class TranscodeService {
bitrate?: string
threads?: number
signal?: AbortSignal
audioStreamIndex?: number
}): Promise<void> {
const bitrate = params.bitrate || '128k'
const args = ['-i', params.inputFile, '-vn', '-c:a', 'aac', '-b:a', bitrate, '-ac', '2']
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
const args = [
'-i',
params.inputFile,
'-vn',
'-map',
aMap,
'-c:a',
'aac',
'-b:a',
bitrate,
'-ac',
'2',
]
if (params.threads && params.threads > 0) {
args.push('-threads', params.threads.toString())
}
@@ -2240,6 +2344,7 @@ export class TranscodeService {
const numFrames = Math.max(1, params.numFrames)
const step = duration / numFrames
const timestamps = Array.from({ length: numFrames }, (_, i) => i * step)
const vMap = meta.videoStreamIndex !== undefined ? `0:${meta.videoStreamIndex}` : '0:V:0'
await mapConcurrent(timestamps, 10, async (t, idx) => {
const outputFile = path.join(params.outputDir, `${idx + 1}.webp`)
@@ -2251,6 +2356,8 @@ export class TranscodeService {
t.toFixed(4),
'-i',
params.inputFile,
'-map',
vMap,
'-vframes',
'1',
'-vf',
@@ -2375,6 +2482,8 @@ export class TranscodeService {
t.toFixed(4),
'-i',
inputSource,
'-map',
'0:V:0',
'-vframes',
'1',
'-vf',
+2
View File
@@ -170,6 +170,8 @@ declare global {
isHdr?: boolean
hdrType?: HdrType
dvProfile?: number
videoStreamIndex?: number
audioStreamIndex?: number
format: unknown
}
@@ -8,11 +8,56 @@ import { assetService } from '@shumai/core/src/asset/asset'
import { fileURLToPath } from 'url'
import * as path from 'path'
import * as fs from 'fs'
import * as os from 'os'
import { execFile } from 'child_process'
import { promisify } from 'util'
const execFileAsync = promisify(execFile)
const currentDir = path.dirname(fileURLToPath(import.meta.url))
const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts')
const fixturesDir = path.resolve(currentDir, '../fixtures')
async function sampleFirstFrameRgb(
buffer: Buffer,
ext: string = 'mp4',
): Promise<{ r: number; g: number; b: number }> {
const tmpFile = path.join(
os.tmpdir(),
`sample-${Date.now()}-${Math.random().toString(36).slice(2)}.${ext}`,
)
try {
fs.writeFileSync(tmpFile, buffer)
const proc = await execFileAsync(
'ffmpeg',
[
'-y',
'-i',
tmpFile,
'-vframes',
'1',
'-s',
'1x1',
'-f',
'rawvideo',
'-pix_fmt',
'rgb24',
'pipe:1',
],
{ encoding: 'buffer' },
)
return {
r: proc.stdout[0],
g: proc.stdout[1],
b: proc.stdout[2],
}
} finally {
if (fs.existsSync(tmpFile)) {
fs.unlinkSync(tmpFile)
}
}
}
describe.each(['local', 'temporal'] as const)(
'Workflow E2E - transcodeVideoWorkflow (executor: %s)',
(mode) => {
@@ -371,5 +416,700 @@ describe.each(['local', 'temporal'] as const)(
})
expect(symlinkVariant).toContain('#EXTM3U')
}, 60000)
it('Case 1: should ignore attached picture cover art (Red) and select primary video stream (Green)', async () => {
const team = await prisma.team.create({
data: { name: 'E2E Cover Art Video Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Cover Art Video Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: { key: 'projects/e2e/case1/video.mp4' },
})
const asset = await prisma.asset.create({
data: {
name: 'video.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case1-'))
const videoPath = path.join(tmpDir, 'case1.mp4')
try {
await execFileAsync('ffmpeg', [
'-y',
'-f',
'lavfi',
'-i',
'color=c=green:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'color=c=red:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'sine=f=1000:d=2',
'-map',
'0:v',
'-map',
'1:v',
'-map',
'2:a',
'-c:v:0',
'libx264',
'-pix_fmt:v:0',
'yuv420p',
'-c:v:1',
'mjpeg',
'-pix_fmt:v:1',
'yuvj420p',
'-c:a',
'aac',
'-disposition:v:1',
'attached_pic',
videoPath,
])
const mp4Buffer = fs.readFileSync(videoPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/case1/video.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true })
}
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
videoStrategy: 'best_match',
poster: true,
sprite: true,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType: string
metadata: { videoStreamIndex: number }
poster: { key: string }
sprite: { key: string }
videoTranscodes: { key: string }[]
}
expect(mediaInfo.proxyType).toBe('video')
expect(mediaInfo.metadata.videoStreamIndex).toBe(0)
// Verify poster is Green (not Red attached pic)
expect(mediaInfo.poster?.key).toBeDefined()
const posterObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.poster.key,
)
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
expect(posterColor.g).toBeGreaterThan(100)
expect(posterColor.r).toBeLessThan(50)
expect(posterColor.b).toBeLessThan(50)
// Verify MP4 proxy is Green
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
const videoObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.videoTranscodes[0].key,
)
const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4')
expect(videoColor.g).toBeGreaterThan(100)
expect(videoColor.r).toBeLessThan(50)
expect(videoColor.b).toBeLessThan(50)
// Verify sprite sheet exists
expect(mediaInfo.sprite?.key).toBeDefined()
await s3Service.headObject('shumai-e2e-test-bucket-transcode', mediaInfo.sprite.key)
}, 60000)
it('Case 2: should select higher bitrate stream (Blue) when both streams have default flag', async () => {
const team = await prisma.team.create({
data: { name: 'E2E Bitrate Comparison Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Bitrate Comparison Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: { key: 'projects/e2e/case2/video.mp4' },
})
const asset = await prisma.asset.create({
data: {
name: 'video.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case2-'))
const videoPath = path.join(tmpDir, 'case2.mp4')
try {
await execFileAsync('ffmpeg', [
'-y',
'-f',
'lavfi',
'-i',
'color=c=red:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'color=c=blue:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'sine=f=1000:d=2',
'-map',
'0:v',
'-map',
'1:v',
'-map',
'2:a',
'-c:v:0',
'libx264',
'-b:v:0',
'200k',
'-minrate:v:0',
'200k',
'-maxrate:v:0',
'200k',
'-bufsize:v:0',
'200k',
'-nal-hrd',
'cbr',
'-disposition:v:0',
'default',
'-c:v:1',
'libx264',
'-b:v:1',
'2000k',
'-minrate:v:1',
'2000k',
'-maxrate:v:1',
'2000k',
'-bufsize:v:1',
'2000k',
'-nal-hrd',
'cbr',
'-disposition:v:1',
'default',
'-c:a',
'aac',
videoPath,
])
const mp4Buffer = fs.readFileSync(videoPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/case2/video.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true })
}
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
videoStrategy: 'best_match',
poster: true,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType: string
metadata: { videoStreamIndex: number }
poster: { key: string }
videoTranscodes: { key: string }[]
}
expect(mediaInfo.proxyType).toBe('video')
// Stream 1 (Blue, 2000k) should be selected over Stream 0 (Red, 200k)
expect(mediaInfo.metadata.videoStreamIndex).toBe(1)
const posterObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.poster.key,
)
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
expect(posterColor.b).toBeGreaterThan(100)
expect(posterColor.r).toBeLessThan(50)
expect(posterColor.g).toBeLessThan(50)
const videoObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.videoTranscodes[0].key,
)
const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4')
expect(videoColor.b).toBeGreaterThan(100)
expect(videoColor.r).toBeLessThan(50)
expect(videoColor.g).toBeLessThan(50)
}, 60000)
it('Case 3: should select stream with default flag (Yellow) over higher bitrate stream without default flag (Blue)', async () => {
const team = await prisma.team.create({
data: { name: 'E2E Default Flag Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Default Flag Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: { key: 'projects/e2e/case3/video.mp4' },
})
const asset = await prisma.asset.create({
data: {
name: 'video.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case3-'))
const videoPath = path.join(tmpDir, 'case3.mp4')
try {
await execFileAsync('ffmpeg', [
'-y',
'-f',
'lavfi',
'-i',
'color=c=yellow:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'color=c=blue:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'sine=f=1000:d=2',
'-map',
'0:v',
'-map',
'1:v',
'-map',
'2:a',
'-c:v:0',
'libx264',
'-b:v:0',
'200k',
'-minrate:v:0',
'200k',
'-maxrate:v:0',
'200k',
'-bufsize:v:0',
'200k',
'-nal-hrd',
'cbr',
'-disposition:v:0',
'default',
'-c:v:1',
'libx264',
'-b:v:1',
'2000k',
'-minrate:v:1',
'2000k',
'-maxrate:v:1',
'2000k',
'-bufsize:v:1',
'2000k',
'-nal-hrd',
'cbr',
'-disposition:v:1',
'0',
'-c:a',
'aac',
videoPath,
])
const mp4Buffer = fs.readFileSync(videoPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/case3/video.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true })
}
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
videoStrategy: 'best_match',
poster: true,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType: string
metadata: { videoStreamIndex: number }
poster: { key: string }
videoTranscodes: { key: string }[]
}
expect(mediaInfo.proxyType).toBe('video')
// Stream 0 (Yellow, default: 1) should win over Stream 1 (Blue, 10x higher bitrate but default: 0)
expect(mediaInfo.metadata.videoStreamIndex).toBe(0)
const posterObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.poster.key,
)
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
expect(posterColor.r).toBeGreaterThan(100)
expect(posterColor.g).toBeGreaterThan(100)
expect(posterColor.b).toBeLessThan(50)
const videoObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.videoTranscodes[0].key,
)
const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4')
expect(videoColor.r).toBeGreaterThan(100)
expect(videoColor.g).toBeGreaterThan(100)
expect(videoColor.b).toBeLessThan(50)
}, 60000)
it('Case 4: should handle audio-only MP4 with cover art by downgrading to audio proxy (catches audio bug)', async () => {
const team = await prisma.team.create({
data: { name: 'E2E Audio Cover Art Team' },
})
const project = await prisma.project.create({
data: { name: 'E2E Audio Cover Art Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: { key: 'projects/e2e/case4/audio-cover.mp4' },
})
// Uploaded as video/mp4 because of .mp4 container, even though it only has audio + cover art
const asset = await prisma.asset.create({
data: {
name: 'audio-cover.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case4-'))
const videoPath = path.join(tmpDir, 'case4.mp4')
try {
await execFileAsync('ffmpeg', [
'-y',
'-f',
'lavfi',
'-i',
'sine=f=1000:d=2',
'-f',
'lavfi',
'-i',
'color=c=magenta:s=320x240:d=2',
'-map',
'0:a',
'-map',
'1:v',
'-c:a',
'aac',
'-c:v',
'mjpeg',
'-pix_fmt:v:0',
'yuvj420p',
'-disposition:v:0',
'attached_pic',
videoPath,
])
const mp4Buffer = fs.readFileSync(videoPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/case4/audio-cover.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true })
}
// Spec requested poster, sprite, videoStrategy as video upload would
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
videoStrategy: 'best_match',
poster: true,
sprite: true,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
// Must complete without error (previously failed trying to generate poster/sprite for audio)
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType: string
poster?: unknown
sprite?: unknown
videoTranscodes: { key: string }[]
}
expect(mediaInfo.proxyType).toBe('audio')
expect(mediaInfo.poster).toBeUndefined()
expect(mediaInfo.sprite).toBeUndefined()
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
expect(mediaInfo.videoTranscodes[0].key).toContain('-audio-proxy.mp4')
await s3Service.headObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.videoTranscodes[0].key,
)
}, 60000)
it('Case 5: should rank 3 video streams with HLS enabled and select highest bitrate stream (Blue)', async () => {
const team = await prisma.team.create({
data: {
name: 'E2E Multi Stream HLS Team',
settings: {
transcode: {
videoStrategy: 'best_match',
hlsEnabled: true,
hlsResolutions: ['480p'],
},
},
},
})
const project = await prisma.project.create({
data: { name: 'E2E Multi Stream HLS Project', teamId: team.id },
})
const storageKey = await prisma.storageKey.create({
data: { key: 'projects/e2e/case5/video.mp4' },
})
const asset = await prisma.asset.create({
data: {
name: 'video.mp4',
type: 'file',
status: 'uploaded',
mediaType: 'video/mp4',
projectId: project.id,
storageKeyId: storageKey.id,
},
})
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case5-'))
const videoPath = path.join(tmpDir, 'case5.mp4')
try {
await execFileAsync('ffmpeg', [
'-y',
'-f',
'lavfi',
'-i',
'color=c=red:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'color=c=green:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'color=c=blue:s=320x240:d=2',
'-f',
'lavfi',
'-i',
'sine=f=1000:d=2',
'-map',
'0:v',
'-map',
'1:v',
'-map',
'2:v',
'-map',
'3:a',
'-c:v:0',
'libx264',
'-b:v:0',
'100k',
'-minrate:v:0',
'100k',
'-maxrate:v:0',
'100k',
'-bufsize:v:0',
'100k',
'-nal-hrd',
'cbr',
'-disposition:v:0',
'default',
'-c:v:1',
'libx264',
'-b:v:1',
'500k',
'-minrate:v:1',
'500k',
'-maxrate:v:1',
'500k',
'-bufsize:v:1',
'500k',
'-nal-hrd',
'cbr',
'-disposition:v:1',
'default',
'-c:v:2',
'libx264',
'-b:v:2',
'1500k',
'-minrate:v:2',
'1500k',
'-maxrate:v:2',
'1500k',
'-bufsize:v:2',
'1500k',
'-nal-hrd',
'cbr',
'-disposition:v:2',
'default',
'-c:a',
'aac',
videoPath,
])
const mp4Buffer = fs.readFileSync(videoPath)
await s3Service.putObject(
'shumai-e2e-test-bucket-transcode',
'projects/e2e/case5/video.mp4',
mp4Buffer,
mp4Buffer.length,
'video/mp4',
)
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true })
}
const task = await prisma.workflowTask.create({
data: {
type: 'transcode_video',
status: 'pending',
assetId: asset.id,
projectId: project.id,
teamId: team.id,
payload: {
projectId: project.id,
transcode: {
videoStrategy: 'best_match',
hlsEnabled: true,
hlsResolutions: ['480p'],
poster: true,
},
},
},
})
const completedTask = await workflowService.executeWait(task, 45000)
expect(completedTask.status).toBe('completed')
const updatedAsset = await prisma.asset.findUnique({
where: { id: asset.id },
})
expect(updatedAsset?.status).toBe(AssetStatus.processed)
const mediaInfo = updatedAsset?.media as unknown as {
proxyType: string
metadata: { videoStreamIndex: number }
poster: { key: string }
isHls: boolean
hls: { key: string; resolutions: { resolution: string }[] }
}
expect(mediaInfo.proxyType).toBe('video')
// Stream 2 (Blue, 1500k) should be selected
expect(mediaInfo.metadata.videoStreamIndex).toBe(2)
const posterObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.poster.key,
)
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
expect(posterColor.b).toBeGreaterThan(100)
expect(posterColor.r).toBeLessThan(50)
expect(posterColor.g).toBeLessThan(50)
expect(mediaInfo.isHls).toBe(true)
const masterObj = await s3Service.getObject(
'shumai-e2e-test-bucket-transcode',
mediaInfo.hls.key,
)
expect(masterObj.buffer.toString('utf-8')).toContain('480p/index.m3u8')
}, 60000)
},
)
@@ -217,6 +217,86 @@ describe('Transcode Activities', () => {
)
})
it('should downgrade proxyType to audio when getVideoInfo throws No video stream found but audio exists', async () => {
const asset = await prisma.asset.create({
data: { name: 'audio-with-art.mp4', type: 'file', status: 'uploaded' },
})
vi.mocked(transcodeService.getVideoInfo).mockRejectedValueOnce(
new Error('No video stream found'),
)
vi.mocked(transcodeService.getAudioInfo).mockResolvedValueOnce({
originalWidth: 0,
originalHeight: 0,
duration: 42,
bitRate: 192000,
frameRate: 0,
totalFrames: 0,
hasAudio: true,
audioCodec: 'aac',
audioChannels: 2,
audioSampleRate: 44100,
audioBitDepth: 16,
audioStreamIndex: 0,
mimeType: 'audio/mp4',
})
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/audio-with-art.mp4',
proxyType: 'video',
mediaType: 'video/mp4',
})
expect(transcodeService.getVideoInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4')
expect(transcodeService.getAudioInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4')
expect(result.proxyType).toBe('audio')
expect(result.duration).toBe(42)
expect(result.metadata?.hasAudio).toBe(true)
expect(result.metadata?.audioStreamIndex).toBe(0)
expect(result.metadata?.videoStreamIndex).toBeUndefined()
expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith(
asset.id,
expect.arrayContaining([
{ key: 'file_type', value: 'audio' },
{ key: 'proxy_type', value: 'audio' },
{ key: 'duration', value: 42 },
]),
true,
)
})
it('should populate videoStreamIndex and audioStreamIndex in mediaInfo.metadata', async () => {
const asset = await prisma.asset.create({
data: { name: 'dual-stream.mp4', type: 'file', status: 'uploaded' },
})
vi.mocked(transcodeService.getVideoInfo).mockResolvedValueOnce({
originalWidth: 1920,
originalHeight: 1080,
duration: 15,
bitRate: 2000000,
frameRate: 30,
totalFrames: 450,
hasAudio: true,
videoCodec: 'h264',
audioCodec: 'aac',
videoStreamIndex: 1,
audioStreamIndex: 2,
mimeType: 'video/mp4',
})
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/dual-stream.mp4',
proxyType: 'video',
mediaType: 'video/mp4',
})
expect(result.metadata?.videoStreamIndex).toBe(1)
expect(result.metadata?.audioStreamIndex).toBe(2)
})
it('should call getImageInfo and set file_type to image', async () => {
const asset = await prisma.asset.create({
data: { name: 'i.png', type: 'file', status: 'uploaded' },
@@ -703,6 +783,28 @@ describe('Transcode Activities', () => {
)
})
it('transcodeVideoActivity forwards streamIndex and audioStreamIndex to transcodeService', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
await transcodeVideoActivity({
assetKey: 'v.mp4',
filePath: '/tmp/v.mp4',
videoSpec: { resolution: '720p', width: 1280, height: 720 },
duration: 10,
originalFps: 30,
streamIndex: 2,
audioStreamIndex: 1,
})
expect(transcodeService.transcodeVideo).toHaveBeenCalledWith(
expect.objectContaining({
inputFile: '/tmp/v.mp4',
streamIndex: 2,
audioStreamIndex: 1,
}),
)
})
it('transcodeVideoActivity appends -hdr.mp4 suffix and forwards HDR metadata', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
@@ -946,6 +1048,44 @@ describe('Transcode Activities', () => {
hdrType: 'pq',
colorTransfer: 'smpte2084',
},
undefined,
)
})
it('generateSpriteActivity forwards videoStreamIndex to transcodeService.generateSprite', async () => {
vi.mocked(transcodeService.generateSprite).mockImplementation(
async (_in, spriteOut, posterOut) => {
const fs = await import('fs')
fs.writeFileSync(spriteOut, 'fake-sprite')
fs.writeFileSync(posterOut, 'fake-poster')
},
)
await generateSpriteActivity({
assetKey: 'files/asset-1/video.mp4',
filePath: '/tmp/video.mp4',
mediaInfo: {
duration: 10,
metadata: {
videoStreamIndex: 1,
},
} as unknown as PrismaJson.MediaInfo,
spriteSpec: { key: 'files/asset-1/sprite.webp', frames: 100, tileX: 10, tileY: 10 },
posterSpec: { key: 'files/asset-1/poster.webp' },
})
expect(transcodeService.generateSprite).toHaveBeenCalledWith(
'/tmp/video.mp4',
expect.any(String),
expect.any(String),
10,
undefined,
{
isHdr: undefined,
hdrType: undefined,
colorTransfer: undefined,
},
1,
)
})
+119 -47
View File
@@ -130,54 +130,110 @@ export async function getMediaInfoActivity(params: {
}
if (isVideo) {
const info = await transcodeService.getVideoInfo(params.filePath)
mediaInfo.duration = info.duration
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: info.duration,
bitRate: info.bitRate,
videoBitRate: info.videoBitRate,
frameRate: info.frameRate,
totalFrames: info.totalFrames,
startTimecode: info.startTimecode || '00:00:00:00',
hasAudio: info.hasAudio,
videoCodec: info.videoCodec,
audioCodec: info.audioCodec,
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
colorTransfer: info.colorTransfer,
colorPrimaries: info.colorPrimaries,
colorSpace: info.colorSpace,
isHdr: info.isHdr,
hdrType: info.hdrType,
dvProfile: info.dvProfile,
rotation: info.rotation,
format: {},
let isDowngradedToAudio = false
let videoInfo: Awaited<ReturnType<typeof transcodeService.getVideoInfo>> | null = null
try {
videoInfo = await transcodeService.getVideoInfo(params.filePath)
} catch (err) {
if (err instanceof Error && err.message.includes('No video stream found')) {
try {
const audioInfo = await transcodeService.getAudioInfo(params.filePath)
isDowngradedToAudio = true
mediaInfo.proxyType = 'audio'
mediaInfo.duration = audioInfo.duration
mediaInfo.metadata = {
originalWidth: 0,
originalHeight: 0,
duration: audioInfo.duration,
bitRate: audioInfo.bitRate,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: true,
videoCodec: undefined,
audioCodec: audioInfo.audioCodec,
audioChannels: audioInfo.audioChannels,
audioSampleRate: audioInfo.audioSampleRate,
audioBitDepth: audioInfo.audioBitDepth,
audioStreamIndex: audioInfo.audioStreamIndex,
format: {},
}
const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type')
if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio'
const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type')
if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio'
metadataUpdates.push(
{ key: 'duration', value: audioInfo.duration },
{ key: 'bitRate', value: audioInfo.bitRate / 1000 },
)
if (audioInfo.audioCodec)
metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec })
if (audioInfo.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels })
if (audioInfo.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate })
if (audioInfo.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth })
} catch {
throw err
}
} else {
throw err
}
}
if (!isDowngradedToAudio && videoInfo) {
const info = videoInfo
mediaInfo.duration = info.duration
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: info.duration,
bitRate: info.bitRate,
videoBitRate: info.videoBitRate,
frameRate: info.frameRate,
totalFrames: info.totalFrames,
startTimecode: info.startTimecode || '00:00:00:00',
hasAudio: info.hasAudio,
videoCodec: info.videoCodec,
audioCodec: info.audioCodec,
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
colorTransfer: info.colorTransfer,
colorPrimaries: info.colorPrimaries,
colorSpace: info.colorSpace,
isHdr: info.isHdr,
hdrType: info.hdrType,
dvProfile: info.dvProfile,
rotation: info.rotation,
videoStreamIndex: info.videoStreamIndex,
audioStreamIndex: info.audioStreamIndex,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'duration', value: info.duration },
{ key: 'bitRate', value: info.bitRate / 1000 },
{ key: 'frame_rate', value: info.frameRate },
)
if (info.rotation !== undefined)
metadataUpdates.push({ key: 'rotation', value: info.rotation })
if (info.isHdr !== undefined)
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
if (info.colorTransfer)
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
if (info.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
if (info.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
if (info.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'duration', value: info.duration },
{ key: 'bitRate', value: info.bitRate / 1000 },
{ key: 'frame_rate', value: info.frameRate },
)
if (info.rotation !== undefined)
metadataUpdates.push({ key: 'rotation', value: info.rotation })
if (info.isHdr !== undefined)
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
if (info.colorTransfer)
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
if (info.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
if (info.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
if (info.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
} else if (isAudio) {
const info = await transcodeService.getAudioInfo(params.filePath)
mediaInfo.duration = info.duration
@@ -195,6 +251,7 @@ export async function getMediaInfoActivity(params: {
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
audioStreamIndex: info.audioStreamIndex,
format: {},
}
metadataUpdates.push(
@@ -277,6 +334,8 @@ export interface VideoActivityParams {
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
streamIndex?: number
audioStreamIndex?: number
}
export async function transcodeVideoActivity(
@@ -338,6 +397,8 @@ export async function transcodeVideoActivity(
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
const stat = fs.statSync(outputFile)
@@ -403,6 +464,8 @@ export interface HlsActivityParams {
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
streamIndex?: number
audioStreamIndex?: number
}
export async function transcodeHlsActivity(params: HlsActivityParams): Promise<PrismaJson.HlsInfo> {
@@ -469,6 +532,8 @@ export async function transcodeHlsActivity(params: HlsActivityParams): Promise<P
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
renditionSpecs.push({
@@ -572,6 +637,7 @@ export interface AudioActivityParams {
assetKey: string
filePath: string
threads?: number
audioStreamIndex?: number
}
export async function transcodeAudioActivity(
@@ -606,6 +672,7 @@ export async function transcodeAudioActivity(
bitrate: '128k',
threads: params.threads,
signal,
audioStreamIndex: params.audioStreamIndex,
})
const stat = fs.statSync(outputFile)
@@ -777,6 +844,7 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
hdrType: params.mediaInfo.metadata?.hdrType,
colorTransfer: params.mediaInfo.metadata?.colorTransfer,
},
params.mediaInfo.metadata?.videoStreamIndex,
)
}
@@ -1081,6 +1149,7 @@ export interface ExtractPosterActivityParams {
isHdr?: boolean
hdrType?: HdrType
colorTransfer?: string
streamIndex?: number
}
export async function extractPosterActivity(
@@ -1112,6 +1181,7 @@ export async function extractPosterActivity(
hdrType: params.hdrType,
colorTransfer: params.colorTransfer,
signal,
streamIndex: params.streamIndex,
})
const posterBuffer = fs.readFileSync(posterFile)
@@ -1428,6 +1498,8 @@ export async function transcodeVideoChunkActivity(
params.startTime.toString(),
'-i',
inputSource,
'-map',
'0:V:0',
'-t',
(params.endTime - params.startTime).toString(),
'-vf',
@@ -370,6 +370,8 @@ export async function transcodeWatermarkMediaActivity(
sourceColorTransfer: colorTransfer ?? originalMedia?.metadata?.colorTransfer,
sourceColorPrimaries: colorPrimaries ?? originalMedia?.metadata?.colorPrimaries,
sourceColorSpace: colorSpace ?? originalMedia?.metadata?.colorSpace,
streamIndex: originalMedia?.metadata?.videoStreamIndex,
audioStreamIndex: originalMedia?.metadata?.audioStreamIndex,
})
const stat = fs.statSync(outFilePath)
@@ -440,6 +442,8 @@ export async function transcodeWatermarkMediaActivity(
sourceColorTransfer: colorTransfer ?? originalMedia?.metadata?.colorTransfer,
sourceColorPrimaries: colorPrimaries ?? originalMedia?.metadata?.colorPrimaries,
sourceColorSpace: colorSpace ?? originalMedia?.metadata?.colorSpace,
streamIndex: originalMedia?.metadata?.videoStreamIndex,
audioStreamIndex: originalMedia?.metadata?.audioStreamIndex,
})
const targetBps = calculateEffectiveBitrateBps(
@@ -64,7 +64,9 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
codec: '',
}
if (spec.poster) {
const isVideo = mediaInfo.proxyType === 'video'
if (spec.poster && isVideo) {
const lastSlashIndex = key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
const posterSpec: PrismaJson.PosterInfo = {
@@ -77,6 +79,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
isHdr: mediaInfo.metadata?.isHdr,
hdrType: mediaInfo.metadata?.hdrType,
colorTransfer: mediaInfo.metadata?.colorTransfer,
streamIndex: mediaInfo.metadata?.videoStreamIndex,
})
mediaInfo.poster = posterResult.poster
await executeActivity(workerQueue, updateAssetMediaActivity, {
@@ -87,7 +90,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
// Generate the sprite (and its poster fallback) before the expensive proxy transcodes so the
// file list can show a poster/sprite preview while the asset is still processing.
if (spec.sprite || (spec.poster && !mediaInfo.poster)) {
if ((spec.sprite || (spec.poster && !mediaInfo.poster)) && isVideo) {
const lastSlashIndex = key.lastIndexOf('/')
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
@@ -124,7 +127,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
const metadata = mediaInfo.metadata
if (mediaInfo.proxyType === 'video' && metadata) {
if (isVideo && metadata) {
const videoResolutions = getTargetVideoResolutions(
spec.videoStrategy || 'best_match',
metadata.originalWidth,
@@ -163,6 +166,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
sourceColorTransfer: metadata.colorTransfer,
sourceColorPrimaries: metadata.colorPrimaries,
sourceColorSpace: metadata.colorSpace,
streamIndex: metadata.videoStreamIndex,
audioStreamIndex: metadata.audioStreamIndex,
})
mediaInfo.videoPreview = videoTranscode
@@ -191,6 +196,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
sourceColorTransfer: metadata.colorTransfer,
sourceColorPrimaries: metadata.colorPrimaries,
sourceColorSpace: metadata.colorSpace,
streamIndex: metadata.videoStreamIndex,
audioStreamIndex: metadata.audioStreamIndex,
})
mediaInfo.videoTranscodes.push(videoTranscode)
@@ -219,6 +226,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
sourceColorTransfer: metadata.colorTransfer,
sourceColorPrimaries: metadata.colorPrimaries,
sourceColorSpace: metadata.colorSpace,
streamIndex: metadata.videoStreamIndex,
audioStreamIndex: metadata.audioStreamIndex,
})
mediaInfo.isHls = true
mediaInfo.hls = hlsResult
@@ -233,11 +242,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
assetKey: key,
filePath,
threads: spec.threads,
audioStreamIndex: metadata.audioStreamIndex,
})
mediaInfo.videoTranscodes.push(audioTranscode)
}
if (spec.thumbnail) {
if (spec.thumbnail && isVideo) {
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
assetKey: key,
filePath,