mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-19 14:16:29 +00:00
fix(transcode): avoid selecting attached picture streams for video transcoding (#443)
* fix(transcode): avoid selecting attached picture streams for video transcoding Filter out attached picture streams (disposition.attached_pic) during video probing and stream selection, matching Immich's default and bitrate selection strategy. Forward explicit primary video and audio stream indices to FFmpeg pipelines to prevent picking embedded cover art or thumbnail streams. * fix(transcode): skip poster and sprite generation for audio-only assets and add multi-stream e2e tests * test(workflow): scope pixel format to h264 stream and use yuvj420p for mjpeg cover art
This commit is contained in:
@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Fixed
|
||||
|
||||
* **transcode**: Fix an issue where videos or audio-only files containing attached cover art or picture streams could cause transcoding to fail during poster generation or select a single static image instead of the video
|
||||
|
||||
* **webui**: Fix inconsistent video resolution labels across download menus, mobile file headers, and the video player by correctly deriving resolutions from the video's short side, including portrait videos and 480p videos
|
||||
|
||||
* **trash**: Fix an issue where the "Empty Trash" confirmation dialog would close immediately instead of showing a loading state while files were being permanently deleted
|
||||
|
||||
@@ -154,6 +154,10 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
|
||||
start.toString(),
|
||||
'-i',
|
||||
inputSource,
|
||||
'-map',
|
||||
'0:V:0',
|
||||
'-map',
|
||||
'0:a:0?',
|
||||
'-t',
|
||||
(end - start).toString(),
|
||||
'-c',
|
||||
|
||||
@@ -12,6 +12,9 @@ import {
|
||||
parseBitrateKbps,
|
||||
buildHlsMasterPlaylist,
|
||||
rewriteM3u8WithPresignedUrls,
|
||||
compareStreams,
|
||||
selectPrimaryVideoStream,
|
||||
selectPrimaryAudioStream,
|
||||
} from './transcode'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
@@ -712,6 +715,8 @@ describe('TranscodeService', () => {
|
||||
'-i',
|
||||
'input.wav',
|
||||
'-vn',
|
||||
'-map',
|
||||
'0:a:0?',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
@@ -746,6 +751,8 @@ describe('TranscodeService', () => {
|
||||
'-i',
|
||||
'input.wav',
|
||||
'-vn',
|
||||
'-map',
|
||||
'0:a:0?',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
@@ -782,6 +789,8 @@ describe('TranscodeService', () => {
|
||||
'-i',
|
||||
'input.wav',
|
||||
'-vn',
|
||||
'-map',
|
||||
'0:a:0?',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
@@ -2831,6 +2840,187 @@ describe('TranscodeService', () => {
|
||||
expect(info.rotation).toBe(180)
|
||||
})
|
||||
|
||||
it('getVideoInfo ignores attached picture streams and selects the main video stream', async () => {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
const mockOutput = JSON.stringify({
|
||||
format: { duration: '10.0', bit_rate: '20000000' },
|
||||
streams: [
|
||||
{
|
||||
index: 0,
|
||||
codec_type: 'video',
|
||||
codec_name: 'mjpeg',
|
||||
width: 320,
|
||||
height: 240,
|
||||
avg_frame_rate: '0/0',
|
||||
r_frame_rate: '90000/1',
|
||||
disposition: {
|
||||
attached_pic: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
codec_type: 'video',
|
||||
codec_name: 'h264',
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
avg_frame_rate: '24/1',
|
||||
r_frame_rate: '24/1',
|
||||
nb_frames: '240',
|
||||
disposition: {
|
||||
default: 1,
|
||||
attached_pic: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
|
||||
mockExecFileStdout(mockOutput)
|
||||
|
||||
const info = await transcodeService.getVideoInfo('cover-art-first.mp4')
|
||||
expect(info.originalWidth).toBe(1920)
|
||||
expect(info.originalHeight).toBe(1080)
|
||||
expect(info.videoStreamIndex).toBe(1)
|
||||
})
|
||||
|
||||
it('getVideoInfo throws error when file only contains audio and attached picture', async () => {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
const mockOutput = JSON.stringify({
|
||||
format: { duration: '10.0', bit_rate: '320000' },
|
||||
streams: [
|
||||
{
|
||||
index: 0,
|
||||
codec_type: 'audio',
|
||||
codec_name: 'aac',
|
||||
channels: 2,
|
||||
sample_rate: '44100',
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
codec_type: 'video',
|
||||
codec_name: 'mjpeg',
|
||||
width: 320,
|
||||
height: 240,
|
||||
disposition: {
|
||||
attached_pic: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
|
||||
mockExecFileStdout(mockOutput)
|
||||
|
||||
await expect(transcodeService.getVideoInfo('audio-with-cover.mp4')).rejects.toThrow(
|
||||
'No video stream found',
|
||||
)
|
||||
})
|
||||
|
||||
it('compareStreams prioritizes default stream then highest bitrate', () => {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
const s1 = { index: 0, bit_rate: '5000000', disposition: { default: 0 } }
|
||||
const s2 = { index: 1, bit_rate: '2000000', disposition: { default: 1 } }
|
||||
const s3 = { index: 2, bit_rate: '8000000', disposition: { default: 0 } }
|
||||
const s4 = { index: 3, disposition: {} }
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
|
||||
const sorted = [s1, s2, s3, s4].sort(compareStreams)
|
||||
expect(sorted[0].index).toBe(1) // default: 1
|
||||
expect(sorted[1].index).toBe(2) // 8000000 bps
|
||||
expect(sorted[2].index).toBe(0) // 5000000 bps
|
||||
expect(sorted[3].index).toBe(3) // 0 bps
|
||||
})
|
||||
|
||||
it('selectPrimaryVideoStream filters out attached picture streams', () => {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
const streams = [
|
||||
{ index: 0, codec_type: 'video', disposition: { attached_pic: 1 }, bit_rate: '9999999' },
|
||||
{ index: 1, codec_type: 'audio', disposition: { default: 1 } },
|
||||
{ index: 2, codec_type: 'video', disposition: { default: 0 }, bit_rate: '1500000' },
|
||||
{ index: 3, codec_type: 'video', disposition: { default: 1 }, bit_rate: '1000000' },
|
||||
]
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
|
||||
const selected = selectPrimaryVideoStream(streams)
|
||||
expect(selected).toBeDefined()
|
||||
expect(selected?.index).toBe(3) // default: 1 non-attached
|
||||
})
|
||||
|
||||
it('selectPrimaryAudioStream picks default then highest bitrate audio stream', () => {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
const streams = [
|
||||
{ index: 0, codec_type: 'video', bit_rate: '2000000' },
|
||||
{ index: 1, codec_type: 'audio', disposition: { default: 0 }, bit_rate: '320000' },
|
||||
{ index: 2, codec_type: 'audio', disposition: { default: 1 }, bit_rate: '128000' },
|
||||
]
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
|
||||
const selected = selectPrimaryAudioStream(streams)
|
||||
expect(selected).toBeDefined()
|
||||
expect(selected?.index).toBe(2) // default: 1 audio
|
||||
})
|
||||
|
||||
it('transcodeVideo uses explicit streamIndex and audioStreamIndex in mapping', async () => {
|
||||
let executedArgs: string[] = []
|
||||
;(
|
||||
child_process.execFile as unknown as {
|
||||
mockImplementation: (
|
||||
fn: (
|
||||
file: string,
|
||||
args: string[],
|
||||
callback: (error: Error | null, result: { stdout: string; stderr: string }) => void,
|
||||
) => void,
|
||||
) => void
|
||||
}
|
||||
).mockImplementation((_file, args, callback) => {
|
||||
executedArgs = args
|
||||
callback(null, { stdout: '', stderr: '' })
|
||||
})
|
||||
|
||||
await transcodeService.transcodeVideo({
|
||||
inputFile: 'input.mp4',
|
||||
outputFile: 'output.mp4',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
hardwareAcceleration: 'off',
|
||||
streamIndex: 2,
|
||||
audioStreamIndex: 1,
|
||||
})
|
||||
|
||||
const filterIdx = executedArgs.indexOf('-filter_complex')
|
||||
expect(filterIdx).toBeGreaterThan(-1)
|
||||
expect(executedArgs[filterIdx + 1]).toContain('[0:2]scale=')
|
||||
|
||||
const mapIndices = executedArgs
|
||||
.map((arg, idx) => (arg === '-map' ? executedArgs[idx + 1] : null))
|
||||
.filter(Boolean)
|
||||
expect(mapIndices).toContain('0:1')
|
||||
})
|
||||
|
||||
it('generatePoster uses explicit streamIndex in mapping', async () => {
|
||||
let executedArgs: string[] = []
|
||||
;(
|
||||
child_process.execFile as unknown as {
|
||||
mockImplementation: (
|
||||
fn: (
|
||||
file: string,
|
||||
args: string[],
|
||||
callback: (error: Error | null, result: { stdout: string; stderr: string }) => void,
|
||||
) => void,
|
||||
) => void
|
||||
}
|
||||
).mockImplementation((_file, args, callback) => {
|
||||
executedArgs = args
|
||||
callback(null, { stdout: '', stderr: '' })
|
||||
})
|
||||
|
||||
await transcodeService.generatePoster('input.mp4', 'poster.webp', { streamIndex: 3 })
|
||||
|
||||
const mapIdx = executedArgs.indexOf('-map')
|
||||
expect(mapIdx).toBeGreaterThan(-1)
|
||||
expect(executedArgs[mapIdx + 1]).toBe('0:3')
|
||||
})
|
||||
|
||||
it('transcodeVideo preserves HDR parameters and tags when hdr is true', async () => {
|
||||
let executedArgs: string[] = []
|
||||
;(
|
||||
@@ -3018,7 +3208,7 @@ describe('TranscodeService', () => {
|
||||
const filterIdx = spriteCmd!.indexOf('-filter_complex')
|
||||
expect(filterIdx).toBeGreaterThan(-1)
|
||||
const filterStr = spriteCmd![filterIdx + 1]
|
||||
expect(filterStr).toContain('[0:v]fps=')
|
||||
expect(filterStr).toContain('[0:V]fps=')
|
||||
expect(filterStr).toContain('scale=w=300:h=-2,tile=10x10[sprite_out]')
|
||||
})
|
||||
|
||||
|
||||
@@ -97,6 +97,54 @@ export interface GeneratePosterOptions {
|
||||
hdrType?: HdrType
|
||||
colorTransfer?: string
|
||||
signal?: AbortSignal
|
||||
streamIndex?: number
|
||||
}
|
||||
|
||||
export interface FfprobeStreamLike {
|
||||
index: number
|
||||
codec_type?: string
|
||||
codec_name?: string
|
||||
bit_rate?: string
|
||||
avg_frame_rate?: string
|
||||
r_frame_rate?: string
|
||||
nb_frames?: string
|
||||
width?: number
|
||||
height?: number
|
||||
channels?: number
|
||||
sample_rate?: string | number
|
||||
bits_per_raw_sample?: string | number
|
||||
bits_per_sample?: string | number
|
||||
color_transfer?: string
|
||||
color_primaries?: string
|
||||
color_space?: string
|
||||
disposition?: Record<string, number | undefined>
|
||||
tags?: Record<string, string | undefined>
|
||||
side_data_list?: Array<Record<string, unknown>>
|
||||
mime_codec_string?: string
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Immich-compatible stream comparator:
|
||||
* 1. Streams with disposition.default: 1 come first.
|
||||
* 2. Ties are broken by higher bit_rate.
|
||||
*/
|
||||
export function compareStreams<T extends FfprobeStreamLike>(a: T, b: T): number {
|
||||
const defDiff = (b.disposition?.default ?? 0) - (a.disposition?.default ?? 0)
|
||||
if (defDiff !== 0) return defDiff
|
||||
const aBitrate = parseInt(a.bit_rate || '0', 10) || 0
|
||||
const bBitrate = parseInt(b.bit_rate || '0', 10) || 0
|
||||
return bBitrate - aBitrate
|
||||
}
|
||||
|
||||
export function selectPrimaryVideoStream<T extends FfprobeStreamLike>(streams: T[]): T | undefined {
|
||||
return streams
|
||||
.filter((s) => s.codec_type === 'video' && !s.disposition?.attached_pic)
|
||||
.sort(compareStreams)[0]
|
||||
}
|
||||
|
||||
export function selectPrimaryAudioStream<T extends FfprobeStreamLike>(streams: T[]): T | undefined {
|
||||
return streams.filter((s) => s.codec_type === 'audio').sort(compareStreams)[0]
|
||||
}
|
||||
|
||||
export interface MediaMetadata {
|
||||
@@ -123,6 +171,8 @@ export interface MediaMetadata {
|
||||
dvProfile?: number
|
||||
dvCompatibilityId?: number
|
||||
rotation?: number
|
||||
videoStreamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
|
||||
export interface TranscodeVideoParams {
|
||||
@@ -144,6 +194,8 @@ export interface TranscodeVideoParams {
|
||||
sourceColorTransfer?: string
|
||||
sourceColorPrimaries?: string
|
||||
sourceColorSpace?: string
|
||||
streamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
|
||||
export interface TranscodeHlsRenditionParams {
|
||||
@@ -166,6 +218,8 @@ export interface TranscodeHlsRenditionParams {
|
||||
sourceColorPrimaries?: string
|
||||
sourceColorSpace?: string
|
||||
segmentDuration?: number
|
||||
streamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
|
||||
export interface EncoderConfig {
|
||||
@@ -475,10 +529,15 @@ export class TranscodeService {
|
||||
return stream.codec_name
|
||||
}
|
||||
|
||||
private safeParseInt(value: string | number | undefined): number | undefined {
|
||||
private safeParseInt(value: unknown): number | undefined {
|
||||
if (value === undefined || value === null) return undefined
|
||||
const parsed = typeof value === 'string' ? parseInt(value, 10) : value
|
||||
return Number.isFinite(parsed) ? (parsed as number) : undefined
|
||||
const parsed =
|
||||
typeof value === 'string'
|
||||
? parseInt(value, 10)
|
||||
: typeof value === 'number'
|
||||
? value
|
||||
: undefined
|
||||
return parsed !== undefined && Number.isFinite(parsed) ? parsed : undefined
|
||||
}
|
||||
|
||||
async getVideoInfo(inputFile: string): Promise<MediaMetadata> {
|
||||
@@ -492,10 +551,8 @@ export class TranscodeService {
|
||||
inputFile,
|
||||
])
|
||||
const info = JSON.parse(stdout)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const videoStream = info.streams.find((s: any) => s.codec_type === 'video')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const audioStream = info.streams.find((s: any) => s.codec_type === 'audio')
|
||||
const videoStream = selectPrimaryVideoStream(info.streams || [])
|
||||
const audioStream = selectPrimaryAudioStream(info.streams || [])
|
||||
|
||||
if (!videoStream) {
|
||||
throw new Error('No video stream found')
|
||||
@@ -678,6 +735,8 @@ export class TranscodeService {
|
||||
hdrType,
|
||||
dvProfile,
|
||||
dvCompatibilityId,
|
||||
videoStreamIndex: videoStream.index,
|
||||
audioStreamIndex: audioStream ? audioStream.index : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1020,11 +1079,13 @@ export class TranscodeService {
|
||||
args.push('-i', params.inputFile)
|
||||
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
|
||||
|
||||
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
|
||||
|
||||
if (params.overlayFile) {
|
||||
args.push('-i', params.overlayFile)
|
||||
filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
} else {
|
||||
filterComplex = `[0:v]${baseScale}`
|
||||
filterComplex = `[${vPad}]${baseScale}`
|
||||
}
|
||||
|
||||
if (params.frameRate) {
|
||||
@@ -1063,7 +1124,8 @@ export class TranscodeService {
|
||||
}
|
||||
|
||||
if (!params.disableAudio) {
|
||||
args.push('-map', '0:a?')
|
||||
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
|
||||
args.push('-map', aMap)
|
||||
}
|
||||
|
||||
args.push('-c:v', encoder.name)
|
||||
@@ -1254,11 +1316,13 @@ export class TranscodeService {
|
||||
args.push('-i', params.inputFile)
|
||||
const baseScale = `scale=w=${params.width}:h=${params.height}:force_original_aspect_ratio=decrease,scale=w='trunc(iw/2)*2':h='trunc(ih/2)*2'`
|
||||
|
||||
const vPad = params.streamIndex !== undefined ? `0:${params.streamIndex}` : '0:V'
|
||||
|
||||
if (params.overlayFile) {
|
||||
args.push('-i', params.overlayFile)
|
||||
filterComplex = `[0:v]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
filterComplex = `[${vPad}]scale=${params.width}:${params.height}[vscaled];[vscaled][1:v]overlay=0:0`
|
||||
} else {
|
||||
filterComplex = `[0:v]${baseScale}`
|
||||
filterComplex = `[${vPad}]${baseScale}`
|
||||
}
|
||||
|
||||
filterComplex += `,fps=${calculatedFps}`
|
||||
@@ -1271,7 +1335,8 @@ export class TranscodeService {
|
||||
args.push('-filter_complex', filterComplex, '-map', '[vout]')
|
||||
|
||||
if (!params.disableAudio) {
|
||||
args.push('-map', '0:a?')
|
||||
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
|
||||
args.push('-map', aMap)
|
||||
}
|
||||
|
||||
args.push('-c:v', encoder.name)
|
||||
@@ -1624,6 +1689,8 @@ export class TranscodeService {
|
||||
|
||||
filters.push('scale=-2:300:force_original_aspect_ratio=decrease')
|
||||
|
||||
const vMap = options.streamIndex !== undefined ? `0:${options.streamIndex}` : '0:V:0'
|
||||
|
||||
const args = [
|
||||
...(isRemote
|
||||
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
|
||||
@@ -1631,6 +1698,8 @@ export class TranscodeService {
|
||||
...(isMpegTs ? [] : ['-skip_frame', 'nointra']),
|
||||
'-i',
|
||||
inputFile,
|
||||
'-map',
|
||||
vMap,
|
||||
'-vf',
|
||||
filters.join(','),
|
||||
'-fps_mode',
|
||||
@@ -1660,13 +1729,14 @@ export class TranscodeService {
|
||||
duration: number,
|
||||
signal?: AbortSignal,
|
||||
hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string },
|
||||
streamIndex?: number,
|
||||
): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
throw new Error('Sprite generation cancelled')
|
||||
}
|
||||
|
||||
// 1. Generate poster first (fast, smart frame selection)
|
||||
await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal })
|
||||
await this.generatePoster(inputFile, outputPoster, { ...hdrOptions, signal, streamIndex })
|
||||
|
||||
// 2. Generate sprite
|
||||
let fileSize = Infinity
|
||||
@@ -1679,9 +1749,23 @@ export class TranscodeService {
|
||||
|
||||
const isSmallAndShort = fileSize <= 50 * 1024 * 1024 && duration <= 30
|
||||
if (isSmallAndShort) {
|
||||
await this.generateSpriteSinglePass(inputFile, outputSprite, duration, signal, hdrOptions)
|
||||
await this.generateSpriteSinglePass(
|
||||
inputFile,
|
||||
outputSprite,
|
||||
duration,
|
||||
signal,
|
||||
hdrOptions,
|
||||
streamIndex,
|
||||
)
|
||||
} else {
|
||||
await this.generateSpriteSeekPool(inputFile, outputSprite, duration, signal, hdrOptions)
|
||||
await this.generateSpriteSeekPool(
|
||||
inputFile,
|
||||
outputSprite,
|
||||
duration,
|
||||
signal,
|
||||
hdrOptions,
|
||||
streamIndex,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1691,9 +1775,11 @@ export class TranscodeService {
|
||||
duration: number,
|
||||
signal?: AbortSignal,
|
||||
hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string },
|
||||
streamIndex?: number,
|
||||
): Promise<void> {
|
||||
const spriteFps = 100 / duration
|
||||
let filterComplex: string
|
||||
const vPad = streamIndex !== undefined ? `0:${streamIndex}` : '0:V'
|
||||
|
||||
if (hdrOptions?.isHdr) {
|
||||
const availableFilters = await this.getAvailableFilters()
|
||||
@@ -1702,9 +1788,9 @@ export class TranscodeService {
|
||||
colorTransfer: hdrOptions.colorTransfer,
|
||||
availableFilters,
|
||||
})
|
||||
filterComplex = `[0:v]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
|
||||
filterComplex = `[${vPad}]${tonemap},fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
|
||||
} else {
|
||||
filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
|
||||
filterComplex = `[${vPad}]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out]`
|
||||
}
|
||||
|
||||
const args = [
|
||||
@@ -1735,6 +1821,7 @@ export class TranscodeService {
|
||||
duration: number,
|
||||
signal?: AbortSignal,
|
||||
hdrOptions?: { isHdr?: boolean; hdrType?: HdrType; colorTransfer?: string },
|
||||
streamIndex?: number,
|
||||
): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
throw new Error('Sprite generation cancelled')
|
||||
@@ -1778,11 +1865,14 @@ export class TranscodeService {
|
||||
|
||||
const ts = timestamps[idx]
|
||||
const framePath = path.join(tmpDir, `frame_${idx.toString().padStart(3, '0')}.webp`)
|
||||
const vMap = streamIndex !== undefined ? `0:${streamIndex}` : '0:V:0'
|
||||
const args = [
|
||||
'-ss',
|
||||
ts.toFixed(3),
|
||||
'-i',
|
||||
inputFile,
|
||||
'-map',
|
||||
vMap,
|
||||
'-vframes',
|
||||
'1',
|
||||
'-vf',
|
||||
@@ -2174,8 +2264,7 @@ export class TranscodeService {
|
||||
inputFile,
|
||||
])
|
||||
const info = JSON.parse(stdout)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const audioStream = info.streams.find((s: any) => s.codec_type === 'audio')
|
||||
const audioStream = selectPrimaryAudioStream(info.streams || [])
|
||||
|
||||
if (!audioStream) {
|
||||
throw new Error('No audio stream found')
|
||||
@@ -2199,6 +2288,7 @@ export class TranscodeService {
|
||||
audioBitDepth:
|
||||
this.safeParseInt(audioStream?.bits_per_raw_sample) ??
|
||||
this.safeParseInt(audioStream?.bits_per_sample),
|
||||
audioStreamIndex: audioStream.index,
|
||||
mimeType: '',
|
||||
}
|
||||
}
|
||||
@@ -2209,9 +2299,23 @@ export class TranscodeService {
|
||||
bitrate?: string
|
||||
threads?: number
|
||||
signal?: AbortSignal
|
||||
audioStreamIndex?: number
|
||||
}): Promise<void> {
|
||||
const bitrate = params.bitrate || '128k'
|
||||
const args = ['-i', params.inputFile, '-vn', '-c:a', 'aac', '-b:a', bitrate, '-ac', '2']
|
||||
const aMap = params.audioStreamIndex !== undefined ? `0:${params.audioStreamIndex}` : '0:a:0?'
|
||||
const args = [
|
||||
'-i',
|
||||
params.inputFile,
|
||||
'-vn',
|
||||
'-map',
|
||||
aMap,
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
bitrate,
|
||||
'-ac',
|
||||
'2',
|
||||
]
|
||||
if (params.threads && params.threads > 0) {
|
||||
args.push('-threads', params.threads.toString())
|
||||
}
|
||||
@@ -2240,6 +2344,7 @@ export class TranscodeService {
|
||||
const numFrames = Math.max(1, params.numFrames)
|
||||
const step = duration / numFrames
|
||||
const timestamps = Array.from({ length: numFrames }, (_, i) => i * step)
|
||||
const vMap = meta.videoStreamIndex !== undefined ? `0:${meta.videoStreamIndex}` : '0:V:0'
|
||||
|
||||
await mapConcurrent(timestamps, 10, async (t, idx) => {
|
||||
const outputFile = path.join(params.outputDir, `${idx + 1}.webp`)
|
||||
@@ -2251,6 +2356,8 @@ export class TranscodeService {
|
||||
t.toFixed(4),
|
||||
'-i',
|
||||
params.inputFile,
|
||||
'-map',
|
||||
vMap,
|
||||
'-vframes',
|
||||
'1',
|
||||
'-vf',
|
||||
@@ -2375,6 +2482,8 @@ export class TranscodeService {
|
||||
t.toFixed(4),
|
||||
'-i',
|
||||
inputSource,
|
||||
'-map',
|
||||
'0:V:0',
|
||||
'-vframes',
|
||||
'1',
|
||||
'-vf',
|
||||
|
||||
@@ -170,6 +170,8 @@ declare global {
|
||||
isHdr?: boolean
|
||||
hdrType?: HdrType
|
||||
dvProfile?: number
|
||||
videoStreamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
format: unknown
|
||||
}
|
||||
|
||||
|
||||
@@ -8,11 +8,56 @@ import { assetService } from '@shumai/core/src/asset/asset'
|
||||
import { fileURLToPath } from 'url'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import { execFile } from 'child_process'
|
||||
import { promisify } from 'util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
const transcodeWorkflowsPath = path.resolve(currentDir, '../../../apps/transcode/src/workflows.ts')
|
||||
const fixturesDir = path.resolve(currentDir, '../fixtures')
|
||||
|
||||
async function sampleFirstFrameRgb(
|
||||
buffer: Buffer,
|
||||
ext: string = 'mp4',
|
||||
): Promise<{ r: number; g: number; b: number }> {
|
||||
const tmpFile = path.join(
|
||||
os.tmpdir(),
|
||||
`sample-${Date.now()}-${Math.random().toString(36).slice(2)}.${ext}`,
|
||||
)
|
||||
try {
|
||||
fs.writeFileSync(tmpFile, buffer)
|
||||
const proc = await execFileAsync(
|
||||
'ffmpeg',
|
||||
[
|
||||
'-y',
|
||||
'-i',
|
||||
tmpFile,
|
||||
'-vframes',
|
||||
'1',
|
||||
'-s',
|
||||
'1x1',
|
||||
'-f',
|
||||
'rawvideo',
|
||||
'-pix_fmt',
|
||||
'rgb24',
|
||||
'pipe:1',
|
||||
],
|
||||
{ encoding: 'buffer' },
|
||||
)
|
||||
return {
|
||||
r: proc.stdout[0],
|
||||
g: proc.stdout[1],
|
||||
b: proc.stdout[2],
|
||||
}
|
||||
} finally {
|
||||
if (fs.existsSync(tmpFile)) {
|
||||
fs.unlinkSync(tmpFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe.each(['local', 'temporal'] as const)(
|
||||
'Workflow E2E - transcodeVideoWorkflow (executor: %s)',
|
||||
(mode) => {
|
||||
@@ -371,5 +416,700 @@ describe.each(['local', 'temporal'] as const)(
|
||||
})
|
||||
expect(symlinkVariant).toContain('#EXTM3U')
|
||||
}, 60000)
|
||||
|
||||
it('Case 1: should ignore attached picture cover art (Red) and select primary video stream (Green)', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Cover Art Video Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Cover Art Video Project', teamId: team.id },
|
||||
})
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'projects/e2e/case1/video.mp4' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case1-'))
|
||||
const videoPath = path.join(tmpDir, 'case1.mp4')
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=green:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=red:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'sine=f=1000:d=2',
|
||||
'-map',
|
||||
'0:v',
|
||||
'-map',
|
||||
'1:v',
|
||||
'-map',
|
||||
'2:a',
|
||||
'-c:v:0',
|
||||
'libx264',
|
||||
'-pix_fmt:v:0',
|
||||
'yuv420p',
|
||||
'-c:v:1',
|
||||
'mjpeg',
|
||||
'-pix_fmt:v:1',
|
||||
'yuvj420p',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-disposition:v:1',
|
||||
'attached_pic',
|
||||
videoPath,
|
||||
])
|
||||
const mp4Buffer = fs.readFileSync(videoPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/case1/video.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
poster: true,
|
||||
sprite: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
metadata: { videoStreamIndex: number }
|
||||
poster: { key: string }
|
||||
sprite: { key: string }
|
||||
videoTranscodes: { key: string }[]
|
||||
}
|
||||
expect(mediaInfo.proxyType).toBe('video')
|
||||
expect(mediaInfo.metadata.videoStreamIndex).toBe(0)
|
||||
|
||||
// Verify poster is Green (not Red attached pic)
|
||||
expect(mediaInfo.poster?.key).toBeDefined()
|
||||
const posterObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.poster.key,
|
||||
)
|
||||
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
|
||||
expect(posterColor.g).toBeGreaterThan(100)
|
||||
expect(posterColor.r).toBeLessThan(50)
|
||||
expect(posterColor.b).toBeLessThan(50)
|
||||
|
||||
// Verify MP4 proxy is Green
|
||||
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
|
||||
const videoObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.videoTranscodes[0].key,
|
||||
)
|
||||
const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4')
|
||||
expect(videoColor.g).toBeGreaterThan(100)
|
||||
expect(videoColor.r).toBeLessThan(50)
|
||||
expect(videoColor.b).toBeLessThan(50)
|
||||
|
||||
// Verify sprite sheet exists
|
||||
expect(mediaInfo.sprite?.key).toBeDefined()
|
||||
await s3Service.headObject('shumai-e2e-test-bucket-transcode', mediaInfo.sprite.key)
|
||||
}, 60000)
|
||||
|
||||
it('Case 2: should select higher bitrate stream (Blue) when both streams have default flag', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Bitrate Comparison Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Bitrate Comparison Project', teamId: team.id },
|
||||
})
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'projects/e2e/case2/video.mp4' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case2-'))
|
||||
const videoPath = path.join(tmpDir, 'case2.mp4')
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=red:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=blue:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'sine=f=1000:d=2',
|
||||
'-map',
|
||||
'0:v',
|
||||
'-map',
|
||||
'1:v',
|
||||
'-map',
|
||||
'2:a',
|
||||
'-c:v:0',
|
||||
'libx264',
|
||||
'-b:v:0',
|
||||
'200k',
|
||||
'-minrate:v:0',
|
||||
'200k',
|
||||
'-maxrate:v:0',
|
||||
'200k',
|
||||
'-bufsize:v:0',
|
||||
'200k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:0',
|
||||
'default',
|
||||
'-c:v:1',
|
||||
'libx264',
|
||||
'-b:v:1',
|
||||
'2000k',
|
||||
'-minrate:v:1',
|
||||
'2000k',
|
||||
'-maxrate:v:1',
|
||||
'2000k',
|
||||
'-bufsize:v:1',
|
||||
'2000k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:1',
|
||||
'default',
|
||||
'-c:a',
|
||||
'aac',
|
||||
videoPath,
|
||||
])
|
||||
const mp4Buffer = fs.readFileSync(videoPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/case2/video.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
poster: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
metadata: { videoStreamIndex: number }
|
||||
poster: { key: string }
|
||||
videoTranscodes: { key: string }[]
|
||||
}
|
||||
expect(mediaInfo.proxyType).toBe('video')
|
||||
// Stream 1 (Blue, 2000k) should be selected over Stream 0 (Red, 200k)
|
||||
expect(mediaInfo.metadata.videoStreamIndex).toBe(1)
|
||||
|
||||
const posterObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.poster.key,
|
||||
)
|
||||
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
|
||||
expect(posterColor.b).toBeGreaterThan(100)
|
||||
expect(posterColor.r).toBeLessThan(50)
|
||||
expect(posterColor.g).toBeLessThan(50)
|
||||
|
||||
const videoObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.videoTranscodes[0].key,
|
||||
)
|
||||
const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4')
|
||||
expect(videoColor.b).toBeGreaterThan(100)
|
||||
expect(videoColor.r).toBeLessThan(50)
|
||||
expect(videoColor.g).toBeLessThan(50)
|
||||
}, 60000)
|
||||
|
||||
it('Case 3: should select stream with default flag (Yellow) over higher bitrate stream without default flag (Blue)', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Default Flag Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Default Flag Project', teamId: team.id },
|
||||
})
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'projects/e2e/case3/video.mp4' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case3-'))
|
||||
const videoPath = path.join(tmpDir, 'case3.mp4')
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=yellow:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=blue:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'sine=f=1000:d=2',
|
||||
'-map',
|
||||
'0:v',
|
||||
'-map',
|
||||
'1:v',
|
||||
'-map',
|
||||
'2:a',
|
||||
'-c:v:0',
|
||||
'libx264',
|
||||
'-b:v:0',
|
||||
'200k',
|
||||
'-minrate:v:0',
|
||||
'200k',
|
||||
'-maxrate:v:0',
|
||||
'200k',
|
||||
'-bufsize:v:0',
|
||||
'200k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:0',
|
||||
'default',
|
||||
'-c:v:1',
|
||||
'libx264',
|
||||
'-b:v:1',
|
||||
'2000k',
|
||||
'-minrate:v:1',
|
||||
'2000k',
|
||||
'-maxrate:v:1',
|
||||
'2000k',
|
||||
'-bufsize:v:1',
|
||||
'2000k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:1',
|
||||
'0',
|
||||
'-c:a',
|
||||
'aac',
|
||||
videoPath,
|
||||
])
|
||||
const mp4Buffer = fs.readFileSync(videoPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/case3/video.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
poster: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
metadata: { videoStreamIndex: number }
|
||||
poster: { key: string }
|
||||
videoTranscodes: { key: string }[]
|
||||
}
|
||||
expect(mediaInfo.proxyType).toBe('video')
|
||||
// Stream 0 (Yellow, default: 1) should win over Stream 1 (Blue, 10x higher bitrate but default: 0)
|
||||
expect(mediaInfo.metadata.videoStreamIndex).toBe(0)
|
||||
|
||||
const posterObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.poster.key,
|
||||
)
|
||||
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
|
||||
expect(posterColor.r).toBeGreaterThan(100)
|
||||
expect(posterColor.g).toBeGreaterThan(100)
|
||||
expect(posterColor.b).toBeLessThan(50)
|
||||
|
||||
const videoObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.videoTranscodes[0].key,
|
||||
)
|
||||
const videoColor = await sampleFirstFrameRgb(videoObj.buffer, 'mp4')
|
||||
expect(videoColor.r).toBeGreaterThan(100)
|
||||
expect(videoColor.g).toBeGreaterThan(100)
|
||||
expect(videoColor.b).toBeLessThan(50)
|
||||
}, 60000)
|
||||
|
||||
it('Case 4: should handle audio-only MP4 with cover art by downgrading to audio proxy (catches audio bug)', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'E2E Audio Cover Art Team' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Audio Cover Art Project', teamId: team.id },
|
||||
})
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'projects/e2e/case4/audio-cover.mp4' },
|
||||
})
|
||||
// Uploaded as video/mp4 because of .mp4 container, even though it only has audio + cover art
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'audio-cover.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case4-'))
|
||||
const videoPath = path.join(tmpDir, 'case4.mp4')
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'sine=f=1000:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=magenta:s=320x240:d=2',
|
||||
'-map',
|
||||
'0:a',
|
||||
'-map',
|
||||
'1:v',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-c:v',
|
||||
'mjpeg',
|
||||
'-pix_fmt:v:0',
|
||||
'yuvj420p',
|
||||
'-disposition:v:0',
|
||||
'attached_pic',
|
||||
videoPath,
|
||||
])
|
||||
const mp4Buffer = fs.readFileSync(videoPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/case4/audio-cover.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
// Spec requested poster, sprite, videoStrategy as video upload would
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
poster: true,
|
||||
sprite: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
// Must complete without error (previously failed trying to generate poster/sprite for audio)
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
poster?: unknown
|
||||
sprite?: unknown
|
||||
videoTranscodes: { key: string }[]
|
||||
}
|
||||
expect(mediaInfo.proxyType).toBe('audio')
|
||||
expect(mediaInfo.poster).toBeUndefined()
|
||||
expect(mediaInfo.sprite).toBeUndefined()
|
||||
expect(mediaInfo.videoTranscodes.length).toBeGreaterThan(0)
|
||||
expect(mediaInfo.videoTranscodes[0].key).toContain('-audio-proxy.mp4')
|
||||
await s3Service.headObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.videoTranscodes[0].key,
|
||||
)
|
||||
}, 60000)
|
||||
|
||||
it('Case 5: should rank 3 video streams with HLS enabled and select highest bitrate stream (Blue)', async () => {
|
||||
const team = await prisma.team.create({
|
||||
data: {
|
||||
name: 'E2E Multi Stream HLS Team',
|
||||
settings: {
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
hlsEnabled: true,
|
||||
hlsResolutions: ['480p'],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'E2E Multi Stream HLS Project', teamId: team.id },
|
||||
})
|
||||
const storageKey = await prisma.storageKey.create({
|
||||
data: { key: 'projects/e2e/case5/video.mp4' },
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'video.mp4',
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'video/mp4',
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
})
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-case5-'))
|
||||
const videoPath = path.join(tmpDir, 'case5.mp4')
|
||||
try {
|
||||
await execFileAsync('ffmpeg', [
|
||||
'-y',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=red:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=green:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'color=c=blue:s=320x240:d=2',
|
||||
'-f',
|
||||
'lavfi',
|
||||
'-i',
|
||||
'sine=f=1000:d=2',
|
||||
'-map',
|
||||
'0:v',
|
||||
'-map',
|
||||
'1:v',
|
||||
'-map',
|
||||
'2:v',
|
||||
'-map',
|
||||
'3:a',
|
||||
'-c:v:0',
|
||||
'libx264',
|
||||
'-b:v:0',
|
||||
'100k',
|
||||
'-minrate:v:0',
|
||||
'100k',
|
||||
'-maxrate:v:0',
|
||||
'100k',
|
||||
'-bufsize:v:0',
|
||||
'100k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:0',
|
||||
'default',
|
||||
'-c:v:1',
|
||||
'libx264',
|
||||
'-b:v:1',
|
||||
'500k',
|
||||
'-minrate:v:1',
|
||||
'500k',
|
||||
'-maxrate:v:1',
|
||||
'500k',
|
||||
'-bufsize:v:1',
|
||||
'500k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:1',
|
||||
'default',
|
||||
'-c:v:2',
|
||||
'libx264',
|
||||
'-b:v:2',
|
||||
'1500k',
|
||||
'-minrate:v:2',
|
||||
'1500k',
|
||||
'-maxrate:v:2',
|
||||
'1500k',
|
||||
'-bufsize:v:2',
|
||||
'1500k',
|
||||
'-nal-hrd',
|
||||
'cbr',
|
||||
'-disposition:v:2',
|
||||
'default',
|
||||
'-c:a',
|
||||
'aac',
|
||||
videoPath,
|
||||
])
|
||||
const mp4Buffer = fs.readFileSync(videoPath)
|
||||
await s3Service.putObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
'projects/e2e/case5/video.mp4',
|
||||
mp4Buffer,
|
||||
mp4Buffer.length,
|
||||
'video/mp4',
|
||||
)
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
type: 'transcode_video',
|
||||
status: 'pending',
|
||||
assetId: asset.id,
|
||||
projectId: project.id,
|
||||
teamId: team.id,
|
||||
payload: {
|
||||
projectId: project.id,
|
||||
transcode: {
|
||||
videoStrategy: 'best_match',
|
||||
hlsEnabled: true,
|
||||
hlsResolutions: ['480p'],
|
||||
poster: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const completedTask = await workflowService.executeWait(task, 45000)
|
||||
expect(completedTask.status).toBe('completed')
|
||||
|
||||
const updatedAsset = await prisma.asset.findUnique({
|
||||
where: { id: asset.id },
|
||||
})
|
||||
expect(updatedAsset?.status).toBe(AssetStatus.processed)
|
||||
|
||||
const mediaInfo = updatedAsset?.media as unknown as {
|
||||
proxyType: string
|
||||
metadata: { videoStreamIndex: number }
|
||||
poster: { key: string }
|
||||
isHls: boolean
|
||||
hls: { key: string; resolutions: { resolution: string }[] }
|
||||
}
|
||||
expect(mediaInfo.proxyType).toBe('video')
|
||||
// Stream 2 (Blue, 1500k) should be selected
|
||||
expect(mediaInfo.metadata.videoStreamIndex).toBe(2)
|
||||
|
||||
const posterObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.poster.key,
|
||||
)
|
||||
const posterColor = await sampleFirstFrameRgb(posterObj.buffer, 'webp')
|
||||
expect(posterColor.b).toBeGreaterThan(100)
|
||||
expect(posterColor.r).toBeLessThan(50)
|
||||
expect(posterColor.g).toBeLessThan(50)
|
||||
|
||||
expect(mediaInfo.isHls).toBe(true)
|
||||
const masterObj = await s3Service.getObject(
|
||||
'shumai-e2e-test-bucket-transcode',
|
||||
mediaInfo.hls.key,
|
||||
)
|
||||
expect(masterObj.buffer.toString('utf-8')).toContain('480p/index.m3u8')
|
||||
}, 60000)
|
||||
},
|
||||
)
|
||||
|
||||
@@ -217,6 +217,86 @@ describe('Transcode Activities', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should downgrade proxyType to audio when getVideoInfo throws No video stream found but audio exists', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'audio-with-art.mp4', type: 'file', status: 'uploaded' },
|
||||
})
|
||||
|
||||
vi.mocked(transcodeService.getVideoInfo).mockRejectedValueOnce(
|
||||
new Error('No video stream found'),
|
||||
)
|
||||
vi.mocked(transcodeService.getAudioInfo).mockResolvedValueOnce({
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
duration: 42,
|
||||
bitRate: 192000,
|
||||
frameRate: 0,
|
||||
totalFrames: 0,
|
||||
hasAudio: true,
|
||||
audioCodec: 'aac',
|
||||
audioChannels: 2,
|
||||
audioSampleRate: 44100,
|
||||
audioBitDepth: 16,
|
||||
audioStreamIndex: 0,
|
||||
mimeType: 'audio/mp4',
|
||||
})
|
||||
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/audio-with-art.mp4',
|
||||
proxyType: 'video',
|
||||
mediaType: 'video/mp4',
|
||||
})
|
||||
|
||||
expect(transcodeService.getVideoInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4')
|
||||
expect(transcodeService.getAudioInfo).toHaveBeenCalledWith('/tmp/audio-with-art.mp4')
|
||||
expect(result.proxyType).toBe('audio')
|
||||
expect(result.duration).toBe(42)
|
||||
expect(result.metadata?.hasAudio).toBe(true)
|
||||
expect(result.metadata?.audioStreamIndex).toBe(0)
|
||||
expect(result.metadata?.videoStreamIndex).toBeUndefined()
|
||||
expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith(
|
||||
asset.id,
|
||||
expect.arrayContaining([
|
||||
{ key: 'file_type', value: 'audio' },
|
||||
{ key: 'proxy_type', value: 'audio' },
|
||||
{ key: 'duration', value: 42 },
|
||||
]),
|
||||
true,
|
||||
)
|
||||
})
|
||||
|
||||
it('should populate videoStreamIndex and audioStreamIndex in mediaInfo.metadata', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'dual-stream.mp4', type: 'file', status: 'uploaded' },
|
||||
})
|
||||
|
||||
vi.mocked(transcodeService.getVideoInfo).mockResolvedValueOnce({
|
||||
originalWidth: 1920,
|
||||
originalHeight: 1080,
|
||||
duration: 15,
|
||||
bitRate: 2000000,
|
||||
frameRate: 30,
|
||||
totalFrames: 450,
|
||||
hasAudio: true,
|
||||
videoCodec: 'h264',
|
||||
audioCodec: 'aac',
|
||||
videoStreamIndex: 1,
|
||||
audioStreamIndex: 2,
|
||||
mimeType: 'video/mp4',
|
||||
})
|
||||
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/dual-stream.mp4',
|
||||
proxyType: 'video',
|
||||
mediaType: 'video/mp4',
|
||||
})
|
||||
|
||||
expect(result.metadata?.videoStreamIndex).toBe(1)
|
||||
expect(result.metadata?.audioStreamIndex).toBe(2)
|
||||
})
|
||||
|
||||
it('should call getImageInfo and set file_type to image', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'i.png', type: 'file', status: 'uploaded' },
|
||||
@@ -703,6 +783,28 @@ describe('Transcode Activities', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('transcodeVideoActivity forwards streamIndex and audioStreamIndex to transcodeService', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
|
||||
await transcodeVideoActivity({
|
||||
assetKey: 'v.mp4',
|
||||
filePath: '/tmp/v.mp4',
|
||||
videoSpec: { resolution: '720p', width: 1280, height: 720 },
|
||||
duration: 10,
|
||||
originalFps: 30,
|
||||
streamIndex: 2,
|
||||
audioStreamIndex: 1,
|
||||
})
|
||||
|
||||
expect(transcodeService.transcodeVideo).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
inputFile: '/tmp/v.mp4',
|
||||
streamIndex: 2,
|
||||
audioStreamIndex: 1,
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('transcodeVideoActivity appends -hdr.mp4 suffix and forwards HDR metadata', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
|
||||
@@ -946,6 +1048,44 @@ describe('Transcode Activities', () => {
|
||||
hdrType: 'pq',
|
||||
colorTransfer: 'smpte2084',
|
||||
},
|
||||
undefined,
|
||||
)
|
||||
})
|
||||
|
||||
it('generateSpriteActivity forwards videoStreamIndex to transcodeService.generateSprite', async () => {
|
||||
vi.mocked(transcodeService.generateSprite).mockImplementation(
|
||||
async (_in, spriteOut, posterOut) => {
|
||||
const fs = await import('fs')
|
||||
fs.writeFileSync(spriteOut, 'fake-sprite')
|
||||
fs.writeFileSync(posterOut, 'fake-poster')
|
||||
},
|
||||
)
|
||||
|
||||
await generateSpriteActivity({
|
||||
assetKey: 'files/asset-1/video.mp4',
|
||||
filePath: '/tmp/video.mp4',
|
||||
mediaInfo: {
|
||||
duration: 10,
|
||||
metadata: {
|
||||
videoStreamIndex: 1,
|
||||
},
|
||||
} as unknown as PrismaJson.MediaInfo,
|
||||
spriteSpec: { key: 'files/asset-1/sprite.webp', frames: 100, tileX: 10, tileY: 10 },
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
})
|
||||
|
||||
expect(transcodeService.generateSprite).toHaveBeenCalledWith(
|
||||
'/tmp/video.mp4',
|
||||
expect.any(String),
|
||||
expect.any(String),
|
||||
10,
|
||||
undefined,
|
||||
{
|
||||
isHdr: undefined,
|
||||
hdrType: undefined,
|
||||
colorTransfer: undefined,
|
||||
},
|
||||
1,
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
@@ -130,54 +130,110 @@ export async function getMediaInfoActivity(params: {
|
||||
}
|
||||
|
||||
if (isVideo) {
|
||||
const info = await transcodeService.getVideoInfo(params.filePath)
|
||||
mediaInfo.duration = info.duration
|
||||
mediaInfo.frames = info.totalFrames
|
||||
mediaInfo.metadata = {
|
||||
originalWidth: info.originalWidth,
|
||||
originalHeight: info.originalHeight,
|
||||
duration: info.duration,
|
||||
bitRate: info.bitRate,
|
||||
videoBitRate: info.videoBitRate,
|
||||
frameRate: info.frameRate,
|
||||
totalFrames: info.totalFrames,
|
||||
startTimecode: info.startTimecode || '00:00:00:00',
|
||||
hasAudio: info.hasAudio,
|
||||
videoCodec: info.videoCodec,
|
||||
audioCodec: info.audioCodec,
|
||||
audioChannels: info.audioChannels,
|
||||
audioSampleRate: info.audioSampleRate,
|
||||
audioBitDepth: info.audioBitDepth,
|
||||
colorTransfer: info.colorTransfer,
|
||||
colorPrimaries: info.colorPrimaries,
|
||||
colorSpace: info.colorSpace,
|
||||
isHdr: info.isHdr,
|
||||
hdrType: info.hdrType,
|
||||
dvProfile: info.dvProfile,
|
||||
rotation: info.rotation,
|
||||
format: {},
|
||||
let isDowngradedToAudio = false
|
||||
let videoInfo: Awaited<ReturnType<typeof transcodeService.getVideoInfo>> | null = null
|
||||
try {
|
||||
videoInfo = await transcodeService.getVideoInfo(params.filePath)
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes('No video stream found')) {
|
||||
try {
|
||||
const audioInfo = await transcodeService.getAudioInfo(params.filePath)
|
||||
isDowngradedToAudio = true
|
||||
mediaInfo.proxyType = 'audio'
|
||||
mediaInfo.duration = audioInfo.duration
|
||||
mediaInfo.metadata = {
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
duration: audioInfo.duration,
|
||||
bitRate: audioInfo.bitRate,
|
||||
frameRate: 0,
|
||||
totalFrames: 0,
|
||||
startTimecode: '00:00:00:00',
|
||||
hasAudio: true,
|
||||
videoCodec: undefined,
|
||||
audioCodec: audioInfo.audioCodec,
|
||||
audioChannels: audioInfo.audioChannels,
|
||||
audioSampleRate: audioInfo.audioSampleRate,
|
||||
audioBitDepth: audioInfo.audioBitDepth,
|
||||
audioStreamIndex: audioInfo.audioStreamIndex,
|
||||
format: {},
|
||||
}
|
||||
const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type')
|
||||
if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio'
|
||||
const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type')
|
||||
if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio'
|
||||
metadataUpdates.push(
|
||||
{ key: 'duration', value: audioInfo.duration },
|
||||
{ key: 'bitRate', value: audioInfo.bitRate / 1000 },
|
||||
)
|
||||
if (audioInfo.audioCodec)
|
||||
metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec })
|
||||
if (audioInfo.audioChannels !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels })
|
||||
if (audioInfo.audioSampleRate !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate })
|
||||
if (audioInfo.audioBitDepth !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth })
|
||||
} catch {
|
||||
throw err
|
||||
}
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
if (!isDowngradedToAudio && videoInfo) {
|
||||
const info = videoInfo
|
||||
mediaInfo.duration = info.duration
|
||||
mediaInfo.frames = info.totalFrames
|
||||
mediaInfo.metadata = {
|
||||
originalWidth: info.originalWidth,
|
||||
originalHeight: info.originalHeight,
|
||||
duration: info.duration,
|
||||
bitRate: info.bitRate,
|
||||
videoBitRate: info.videoBitRate,
|
||||
frameRate: info.frameRate,
|
||||
totalFrames: info.totalFrames,
|
||||
startTimecode: info.startTimecode || '00:00:00:00',
|
||||
hasAudio: info.hasAudio,
|
||||
videoCodec: info.videoCodec,
|
||||
audioCodec: info.audioCodec,
|
||||
audioChannels: info.audioChannels,
|
||||
audioSampleRate: info.audioSampleRate,
|
||||
audioBitDepth: info.audioBitDepth,
|
||||
colorTransfer: info.colorTransfer,
|
||||
colorPrimaries: info.colorPrimaries,
|
||||
colorSpace: info.colorSpace,
|
||||
isHdr: info.isHdr,
|
||||
hdrType: info.hdrType,
|
||||
dvProfile: info.dvProfile,
|
||||
rotation: info.rotation,
|
||||
videoStreamIndex: info.videoStreamIndex,
|
||||
audioStreamIndex: info.audioStreamIndex,
|
||||
format: {},
|
||||
}
|
||||
metadataUpdates.push(
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
{ key: 'duration', value: info.duration },
|
||||
{ key: 'bitRate', value: info.bitRate / 1000 },
|
||||
{ key: 'frame_rate', value: info.frameRate },
|
||||
)
|
||||
if (info.rotation !== undefined)
|
||||
metadataUpdates.push({ key: 'rotation', value: info.rotation })
|
||||
if (info.isHdr !== undefined)
|
||||
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
|
||||
if (info.colorTransfer)
|
||||
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
|
||||
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
|
||||
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
|
||||
if (info.audioChannels !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
|
||||
if (info.audioSampleRate !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
|
||||
if (info.audioBitDepth !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
|
||||
}
|
||||
metadataUpdates.push(
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
{ key: 'duration', value: info.duration },
|
||||
{ key: 'bitRate', value: info.bitRate / 1000 },
|
||||
{ key: 'frame_rate', value: info.frameRate },
|
||||
)
|
||||
if (info.rotation !== undefined)
|
||||
metadataUpdates.push({ key: 'rotation', value: info.rotation })
|
||||
if (info.isHdr !== undefined)
|
||||
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
|
||||
if (info.colorTransfer)
|
||||
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
|
||||
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
|
||||
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
|
||||
if (info.audioChannels !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
|
||||
if (info.audioSampleRate !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
|
||||
if (info.audioBitDepth !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
|
||||
} else if (isAudio) {
|
||||
const info = await transcodeService.getAudioInfo(params.filePath)
|
||||
mediaInfo.duration = info.duration
|
||||
@@ -195,6 +251,7 @@ export async function getMediaInfoActivity(params: {
|
||||
audioChannels: info.audioChannels,
|
||||
audioSampleRate: info.audioSampleRate,
|
||||
audioBitDepth: info.audioBitDepth,
|
||||
audioStreamIndex: info.audioStreamIndex,
|
||||
format: {},
|
||||
}
|
||||
metadataUpdates.push(
|
||||
@@ -277,6 +334,8 @@ export interface VideoActivityParams {
|
||||
sourceColorTransfer?: string
|
||||
sourceColorPrimaries?: string
|
||||
sourceColorSpace?: string
|
||||
streamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
|
||||
export async function transcodeVideoActivity(
|
||||
@@ -338,6 +397,8 @@ export async function transcodeVideoActivity(
|
||||
sourceColorTransfer: params.sourceColorTransfer,
|
||||
sourceColorPrimaries: params.sourceColorPrimaries,
|
||||
sourceColorSpace: params.sourceColorSpace,
|
||||
streamIndex: params.streamIndex,
|
||||
audioStreamIndex: params.audioStreamIndex,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outputFile)
|
||||
@@ -403,6 +464,8 @@ export interface HlsActivityParams {
|
||||
sourceColorTransfer?: string
|
||||
sourceColorPrimaries?: string
|
||||
sourceColorSpace?: string
|
||||
streamIndex?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
|
||||
export async function transcodeHlsActivity(params: HlsActivityParams): Promise<PrismaJson.HlsInfo> {
|
||||
@@ -469,6 +532,8 @@ export async function transcodeHlsActivity(params: HlsActivityParams): Promise<P
|
||||
sourceColorTransfer: params.sourceColorTransfer,
|
||||
sourceColorPrimaries: params.sourceColorPrimaries,
|
||||
sourceColorSpace: params.sourceColorSpace,
|
||||
streamIndex: params.streamIndex,
|
||||
audioStreamIndex: params.audioStreamIndex,
|
||||
})
|
||||
|
||||
renditionSpecs.push({
|
||||
@@ -572,6 +637,7 @@ export interface AudioActivityParams {
|
||||
assetKey: string
|
||||
filePath: string
|
||||
threads?: number
|
||||
audioStreamIndex?: number
|
||||
}
|
||||
|
||||
export async function transcodeAudioActivity(
|
||||
@@ -606,6 +672,7 @@ export async function transcodeAudioActivity(
|
||||
bitrate: '128k',
|
||||
threads: params.threads,
|
||||
signal,
|
||||
audioStreamIndex: params.audioStreamIndex,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outputFile)
|
||||
@@ -777,6 +844,7 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
hdrType: params.mediaInfo.metadata?.hdrType,
|
||||
colorTransfer: params.mediaInfo.metadata?.colorTransfer,
|
||||
},
|
||||
params.mediaInfo.metadata?.videoStreamIndex,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1081,6 +1149,7 @@ export interface ExtractPosterActivityParams {
|
||||
isHdr?: boolean
|
||||
hdrType?: HdrType
|
||||
colorTransfer?: string
|
||||
streamIndex?: number
|
||||
}
|
||||
|
||||
export async function extractPosterActivity(
|
||||
@@ -1112,6 +1181,7 @@ export async function extractPosterActivity(
|
||||
hdrType: params.hdrType,
|
||||
colorTransfer: params.colorTransfer,
|
||||
signal,
|
||||
streamIndex: params.streamIndex,
|
||||
})
|
||||
|
||||
const posterBuffer = fs.readFileSync(posterFile)
|
||||
@@ -1428,6 +1498,8 @@ export async function transcodeVideoChunkActivity(
|
||||
params.startTime.toString(),
|
||||
'-i',
|
||||
inputSource,
|
||||
'-map',
|
||||
'0:V:0',
|
||||
'-t',
|
||||
(params.endTime - params.startTime).toString(),
|
||||
'-vf',
|
||||
|
||||
@@ -370,6 +370,8 @@ export async function transcodeWatermarkMediaActivity(
|
||||
sourceColorTransfer: colorTransfer ?? originalMedia?.metadata?.colorTransfer,
|
||||
sourceColorPrimaries: colorPrimaries ?? originalMedia?.metadata?.colorPrimaries,
|
||||
sourceColorSpace: colorSpace ?? originalMedia?.metadata?.colorSpace,
|
||||
streamIndex: originalMedia?.metadata?.videoStreamIndex,
|
||||
audioStreamIndex: originalMedia?.metadata?.audioStreamIndex,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outFilePath)
|
||||
@@ -440,6 +442,8 @@ export async function transcodeWatermarkMediaActivity(
|
||||
sourceColorTransfer: colorTransfer ?? originalMedia?.metadata?.colorTransfer,
|
||||
sourceColorPrimaries: colorPrimaries ?? originalMedia?.metadata?.colorPrimaries,
|
||||
sourceColorSpace: colorSpace ?? originalMedia?.metadata?.colorSpace,
|
||||
streamIndex: originalMedia?.metadata?.videoStreamIndex,
|
||||
audioStreamIndex: originalMedia?.metadata?.audioStreamIndex,
|
||||
})
|
||||
|
||||
const targetBps = calculateEffectiveBitrateBps(
|
||||
|
||||
@@ -64,7 +64,9 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
codec: '',
|
||||
}
|
||||
|
||||
if (spec.poster) {
|
||||
const isVideo = mediaInfo.proxyType === 'video'
|
||||
|
||||
if (spec.poster && isVideo) {
|
||||
const lastSlashIndex = key.lastIndexOf('/')
|
||||
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
|
||||
const posterSpec: PrismaJson.PosterInfo = {
|
||||
@@ -77,6 +79,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
isHdr: mediaInfo.metadata?.isHdr,
|
||||
hdrType: mediaInfo.metadata?.hdrType,
|
||||
colorTransfer: mediaInfo.metadata?.colorTransfer,
|
||||
streamIndex: mediaInfo.metadata?.videoStreamIndex,
|
||||
})
|
||||
mediaInfo.poster = posterResult.poster
|
||||
await executeActivity(workerQueue, updateAssetMediaActivity, {
|
||||
@@ -87,7 +90,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
|
||||
// Generate the sprite (and its poster fallback) before the expensive proxy transcodes so the
|
||||
// file list can show a poster/sprite preview while the asset is still processing.
|
||||
if (spec.sprite || (spec.poster && !mediaInfo.poster)) {
|
||||
if ((spec.sprite || (spec.poster && !mediaInfo.poster)) && isVideo) {
|
||||
const lastSlashIndex = key.lastIndexOf('/')
|
||||
const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex)
|
||||
|
||||
@@ -124,7 +127,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
|
||||
const metadata = mediaInfo.metadata
|
||||
|
||||
if (mediaInfo.proxyType === 'video' && metadata) {
|
||||
if (isVideo && metadata) {
|
||||
const videoResolutions = getTargetVideoResolutions(
|
||||
spec.videoStrategy || 'best_match',
|
||||
metadata.originalWidth,
|
||||
@@ -163,6 +166,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
sourceColorTransfer: metadata.colorTransfer,
|
||||
sourceColorPrimaries: metadata.colorPrimaries,
|
||||
sourceColorSpace: metadata.colorSpace,
|
||||
streamIndex: metadata.videoStreamIndex,
|
||||
audioStreamIndex: metadata.audioStreamIndex,
|
||||
})
|
||||
|
||||
mediaInfo.videoPreview = videoTranscode
|
||||
@@ -191,6 +196,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
sourceColorTransfer: metadata.colorTransfer,
|
||||
sourceColorPrimaries: metadata.colorPrimaries,
|
||||
sourceColorSpace: metadata.colorSpace,
|
||||
streamIndex: metadata.videoStreamIndex,
|
||||
audioStreamIndex: metadata.audioStreamIndex,
|
||||
})
|
||||
|
||||
mediaInfo.videoTranscodes.push(videoTranscode)
|
||||
@@ -219,6 +226,8 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
sourceColorTransfer: metadata.colorTransfer,
|
||||
sourceColorPrimaries: metadata.colorPrimaries,
|
||||
sourceColorSpace: metadata.colorSpace,
|
||||
streamIndex: metadata.videoStreamIndex,
|
||||
audioStreamIndex: metadata.audioStreamIndex,
|
||||
})
|
||||
mediaInfo.isHls = true
|
||||
mediaInfo.hls = hlsResult
|
||||
@@ -233,11 +242,12 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
assetKey: key,
|
||||
filePath,
|
||||
threads: spec.threads,
|
||||
audioStreamIndex: metadata.audioStreamIndex,
|
||||
})
|
||||
mediaInfo.videoTranscodes.push(audioTranscode)
|
||||
}
|
||||
|
||||
if (spec.thumbnail) {
|
||||
if (spec.thumbnail && isVideo) {
|
||||
const thumbTranscode = await executeActivity(workerQueue, transcodeImageActivity, {
|
||||
assetKey: key,
|
||||
filePath,
|
||||
|
||||
Reference in New Issue
Block a user