Files
shumai/packages/transcode/src/activities/transcode.ts
T
Yiling-J 3564d4d164 fix(transcode): avoid selecting attached picture streams for video transcoding (#443)
* fix(transcode): avoid selecting attached picture streams for video transcoding

Filter out attached picture streams (disposition.attached_pic) during video probing and stream selection, matching Immich's default and bitrate selection strategy. Forward explicit primary video and audio stream indices to FFmpeg pipelines to prevent picking embedded cover art or thumbnail streams.

* fix(transcode): skip poster and sprite generation for audio-only assets and add multi-stream e2e tests

* test(workflow): scope pixel format to h264 stream and use yuvj420p for mjpeg cover art
2026-10-02 13:51:17 +08:00

1548 lines
46 KiB
TypeScript

import { AssetStatus, prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import {
transcodeService,
buildHlsMasterPlaylist,
calculateEffectiveBitrateBps,
type HdrType,
} from '@shumai/core/src/transcode/transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { getDerivedArtifactDirectory, stemFromKey } from '@shumai/core/src/utils/filename'
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
import { parseCsvContent } from '@shumai/core/src/transcode/transcode'
import { resolutionToDimensions } from '../workflows/transcode-utils'
import {
getProxyType,
isCsvDocument,
isHtmlDocument,
isMarkdownDocument,
isOfficeDocument,
} from '@shumai/core/src/utils/mime'
import { logger } from '@shumai/core/src/logger'
import { ApplicationFailure, Context } from '@temporalio/activity'
import { getLocalTaskAbortSignal } from '@shumai/workflow-core'
function getActivityCancellationSignal(taskId?: string): AbortSignal | undefined {
try {
return Context.current().cancellationSignal
} catch {
return getLocalTaskAbortSignal(taskId)
}
}
import * as path from 'path'
import * as fs from 'fs'
import * as os from 'os'
import { execFile } from 'child_process'
import { promisify } from 'util'
import { ulid } from 'ulid'
const execFileAsync = promisify(execFile)
export interface GetMediaInfoActivityParams {
assetId: string
assetKey: string
mediaType: string
}
function getErrorDetails(err: unknown): { code?: string; message: string; name?: string } {
if (err && typeof err === 'object') {
const record = err as Record<string, unknown>
return {
code: typeof record.code === 'string' ? record.code : undefined,
message:
typeof record.message === 'string'
? record.message
: err instanceof Error
? err.message
: String(err),
name: typeof record.name === 'string' ? record.name : undefined,
}
}
return {
message: String(err),
}
}
async function ensureAssetNotPurging(assetKey: string): Promise<void> {
const sk = await prisma.storageKey.findUnique({
where: { key: assetKey },
include: { assets: { select: { id: true, status: true } } },
})
if (
sk &&
(sk.status === 'purging' ||
sk.assets.length === 0 ||
sk.assets.every((a) => a.status === AssetStatus.pending_purge))
) {
throw ApplicationFailure.create({
message: 'Asset or storage key has been purged or is pending purge',
nonRetryable: true,
})
}
}
export async function getMediaInfoActivity(params: {
filePath: string
assetId: string
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
mediaType?: string
}): Promise<PrismaJson.MediaInfo> {
try {
const proxyType =
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
const fileType = isVideo
? 'video'
: isAudio
? 'audio'
: isImage
? 'image'
: isPdf
? 'document'
: 'file'
const mediaInfo: PrismaJson.MediaInfo = {
duration: 0,
filesize: 0,
frames: 0,
proxyType,
imageTranscodes: [],
videoTranscodes: [],
videoPreview: { width: 0, height: 0 },
finishedAt: new Date().toISOString(),
metadata: null,
original: {
key: '', // Will be filled by caller or updated later
filesizeInBytes: 0,
codec: '',
},
}
const metadataUpdates: { key: string; value: string | number }[] = [
{ key: 'file_type', value: fileType },
]
if (proxyType) {
metadataUpdates.push({ key: 'proxy_type', value: proxyType })
}
if (isVideo) {
let isDowngradedToAudio = false
let videoInfo: Awaited<ReturnType<typeof transcodeService.getVideoInfo>> | null = null
try {
videoInfo = await transcodeService.getVideoInfo(params.filePath)
} catch (err) {
if (err instanceof Error && err.message.includes('No video stream found')) {
try {
const audioInfo = await transcodeService.getAudioInfo(params.filePath)
isDowngradedToAudio = true
mediaInfo.proxyType = 'audio'
mediaInfo.duration = audioInfo.duration
mediaInfo.metadata = {
originalWidth: 0,
originalHeight: 0,
duration: audioInfo.duration,
bitRate: audioInfo.bitRate,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: true,
videoCodec: undefined,
audioCodec: audioInfo.audioCodec,
audioChannels: audioInfo.audioChannels,
audioSampleRate: audioInfo.audioSampleRate,
audioBitDepth: audioInfo.audioBitDepth,
audioStreamIndex: audioInfo.audioStreamIndex,
format: {},
}
const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type')
if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio'
const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type')
if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio'
metadataUpdates.push(
{ key: 'duration', value: audioInfo.duration },
{ key: 'bitRate', value: audioInfo.bitRate / 1000 },
)
if (audioInfo.audioCodec)
metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec })
if (audioInfo.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels })
if (audioInfo.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate })
if (audioInfo.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth })
} catch {
throw err
}
} else {
throw err
}
}
if (!isDowngradedToAudio && videoInfo) {
const info = videoInfo
mediaInfo.duration = info.duration
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: info.duration,
bitRate: info.bitRate,
videoBitRate: info.videoBitRate,
frameRate: info.frameRate,
totalFrames: info.totalFrames,
startTimecode: info.startTimecode || '00:00:00:00',
hasAudio: info.hasAudio,
videoCodec: info.videoCodec,
audioCodec: info.audioCodec,
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
colorTransfer: info.colorTransfer,
colorPrimaries: info.colorPrimaries,
colorSpace: info.colorSpace,
isHdr: info.isHdr,
hdrType: info.hdrType,
dvProfile: info.dvProfile,
rotation: info.rotation,
videoStreamIndex: info.videoStreamIndex,
audioStreamIndex: info.audioStreamIndex,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'duration', value: info.duration },
{ key: 'bitRate', value: info.bitRate / 1000 },
{ key: 'frame_rate', value: info.frameRate },
)
if (info.rotation !== undefined)
metadataUpdates.push({ key: 'rotation', value: info.rotation })
if (info.isHdr !== undefined)
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
if (info.colorTransfer)
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
if (info.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
if (info.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
if (info.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
}
} else if (isAudio) {
const info = await transcodeService.getAudioInfo(params.filePath)
mediaInfo.duration = info.duration
mediaInfo.metadata = {
originalWidth: 0,
originalHeight: 0,
duration: info.duration,
bitRate: info.bitRate,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: true,
videoCodec: undefined,
audioCodec: info.audioCodec,
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
audioStreamIndex: info.audioStreamIndex,
format: {},
}
metadataUpdates.push(
{ key: 'duration', value: info.duration },
{ key: 'bitRate', value: info.bitRate / 1000 },
)
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
if (info.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
if (info.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
if (info.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
} else if (isImage) {
const info = await transcodeService.getImageInfo(params.filePath)
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: false,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
)
} else if (isPdf) {
const info = await transcodeService.getPdfInfo(params.filePath)
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: info.totalFrames,
startTimecode: '00:00:00:00',
hasAudio: false,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'total_pages', value: info.totalFrames },
)
}
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
return mediaInfo
} catch (err) {
const { name, message } = getErrorDetails(err)
if (name?.includes('Prisma') || message.includes('Prisma')) {
throw err
}
throw ApplicationFailure.create({
message: `Failed to get media info: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
}
export interface VideoActivityParams {
taskId?: string
assetKey: string
filePath: string
videoSpec: PrismaJson.VideoTranscode
duration: number
originalFps: number
hardwareAcceleration?: PrismaJson.HardwareAcceleration
sourceVideoBitrate?: number
threads?: number
sourceIsHdr?: boolean
sourceHdrType?: HdrType
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
streamIndex?: number
audioStreamIndex?: number
}
export async function transcodeVideoActivity(
params: VideoActivityParams,
): Promise<PrismaJson.VideoTranscode> {
const bucket = process.env.S3_BUCKET || 'shumai'
const stem = stemFromKey(params.assetKey)
const res = params.videoSpec.resolution || `${params.videoSpec.height}p`
const suffix = params.videoSpec.hdr ? `${res}-hdr` : res
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-${suffix}.mp4`)
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Video transcoding cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, key)
return { ...params.videoSpec, key }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const outputFile = path.join(tmpDir, `${stem}-${suffix}.mp4`)
try {
let targetFps: number | string = params.originalFps || 30
let disableAudio = false
if (params.videoSpec.resolution === '180p') {
disableAudio = true
targetFps = 24.0
if (params.originalFps > 0 && params.originalFps < 24.0) {
targetFps = params.originalFps
}
if (params.duration * (targetFps as number) > 1600) {
targetFps = `160000/${Math.floor(params.duration * 100)}`
}
}
await transcodeService.transcodeVideo({
inputFile: params.filePath,
outputFile,
width: params.videoSpec.width,
height: params.videoSpec.height,
frameRate: targetFps || undefined,
disableAudio,
hardwareAcceleration: params.hardwareAcceleration,
sourceVideoBitrate: params.sourceVideoBitrate,
threads: params.threads,
signal,
hdr: params.videoSpec.hdr,
sourceIsHdr: params.sourceIsHdr,
sourceHdrType: params.sourceHdrType,
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
const stat = fs.statSync(outputFile)
const stream = Bun.file(outputFile).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
return { ...params.videoSpec, key }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Video transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('no video stream found') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `Video transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
}
}
export interface HlsActivityParams {
taskId: string
assetKey: string
filePath: string
ladders: PrismaJson.HlsResolutionLadder[]
originalWidth: number
originalHeight: number
duration: number
originalFps: number
hardwareAcceleration?: 'off' | 'auto'
sourceVideoBitrate?: number
threads?: number
sourceIsHdr?: boolean
sourceHdrType?: HdrType
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
streamIndex?: number
audioStreamIndex?: number
}
export async function transcodeHlsActivity(params: HlsActivityParams): Promise<PrismaJson.HlsInfo> {
const bucket = process.env.S3_BUCKET || 'shumai'
const assetDir = path.posix.dirname(params.assetKey)
const hlsDirKey = path.posix.join(assetDir, 'hls')
const masterKey = path.posix.join(hlsDirKey, 'master.m3u8')
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'HLS transcoding cancelled',
nonRetryable: true,
})
}
const hlsTmpDir = path.join(path.dirname(params.filePath), `hls-${ulid()}`)
try {
const renditionSpecs: Array<{
resolution: string
width: number
height: number
bitrateBps: number
isHdr?: boolean
}> = []
const mediaResolutions: PrismaJson.VideoTranscode[] = []
for (const ladder of params.ladders) {
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'HLS transcoding cancelled',
nonRetryable: true,
})
}
const [width, height] = resolutionToDimensions(
ladder,
params.originalWidth,
params.originalHeight,
)
const targetBps = calculateEffectiveBitrateBps(
height,
width,
params.sourceVideoBitrate,
params.originalFps,
)
const ladderDir = path.join(hlsTmpDir, ladder)
await transcodeService.transcodeHlsRendition({
inputFile: params.filePath,
outputDir: ladderDir,
width,
height,
frameRate: params.originalFps,
hardwareAcceleration: params.hardwareAcceleration,
sourceVideoBitrate: params.sourceVideoBitrate,
threads: params.threads,
signal,
hdr: Boolean(params.sourceIsHdr),
sourceIsHdr: params.sourceIsHdr,
sourceHdrType: params.sourceHdrType,
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
renditionSpecs.push({
resolution: ladder,
width,
height,
bitrateBps: targetBps,
isHdr: params.sourceIsHdr,
})
mediaResolutions.push({
resolution: ladder,
width,
height,
hdr: Boolean(params.sourceIsHdr),
})
await ensureAssetNotPurging(params.assetKey)
const files = fs.readdirSync(ladderDir)
for (const file of files) {
const localFilePath = path.join(ladderDir, file)
const stat = fs.statSync(localFilePath)
const s3Key = path.posix.join(hlsDirKey, ladder, file)
let contentType = 'application/octet-stream'
if (file.endsWith('.m3u8')) {
contentType = 'application/vnd.apple.mpegurl'
} else if (file.endsWith('.mp4')) {
contentType = 'video/mp4'
} else if (file.endsWith('.m4s')) {
contentType = 'video/iso.segment'
}
const stream = Bun.file(localFilePath).stream()
await s3Service.putObject(bucket, s3Key, stream, stat.size, contentType)
}
}
const masterContent = buildHlsMasterPlaylist(renditionSpecs)
const masterBuffer = Buffer.from(masterContent, 'utf-8')
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
masterKey,
masterBuffer,
masterBuffer.length,
'application/vnd.apple.mpegurl',
)
return {
key: masterKey,
resolutions: mediaResolutions,
}
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'HLS transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('no video stream found') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `HLS transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(hlsTmpDir)) {
try {
fs.rmSync(hlsTmpDir, { recursive: true, force: true })
} catch {
// ignore cleanup error
}
}
}
}
export interface AudioActivityParams {
taskId?: string
assetKey: string
filePath: string
threads?: number
audioStreamIndex?: number
}
export async function transcodeAudioActivity(
params: AudioActivityParams,
): Promise<PrismaJson.VideoTranscode> {
const bucket = process.env.S3_BUCKET || 'shumai'
const stem = stemFromKey(params.assetKey)
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Audio transcoding cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, key)
return { width: 0, height: 0, key }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const outputFile = path.join(tmpDir, `${stem}-audio-proxy.mp4`)
try {
await transcodeService.transcodeAudio({
inputFile: params.filePath,
outputFile,
bitrate: '128k',
threads: params.threads,
signal,
audioStreamIndex: params.audioStreamIndex,
})
const stat = fs.statSync(outputFile)
const stream = Bun.file(outputFile).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
return { width: 0, height: 0, key }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Audio transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('no video stream found')
) {
throw ApplicationFailure.create({
message: `Audio transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
}
}
export interface ImageActivityParams {
assetKey: string
filePath: string
imageSpec: PrismaJson.ImageTranscode
}
export async function transcodeImageActivity(
params: ImageActivityParams,
): Promise<PrismaJson.ImageTranscode> {
const bucket = process.env.S3_BUCKET || 'shumai'
const stem = stemFromKey(params.assetKey)
const key = path.posix.join(
path.posix.dirname(params.assetKey),
`${stem}-${params.imageSpec.width}p.webp`,
)
try {
await s3Service.headObject(bucket, key)
return { ...params.imageSpec, key }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const outputFile = path.join(tmpDir, `${stem}-${params.imageSpec.width}p.webp`)
try {
await transcodeService.transcodeImage(
params.filePath,
outputFile,
params.imageSpec.width,
params.imageSpec.quality,
{
height: params.imageSpec.height || null,
isPreview: params.imageSpec.isPreview ?? false,
},
)
const buffer = fs.readFileSync(outputFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
return { ...params.imageSpec, key }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('sharp') ||
lowerMsg.includes('imagemagick') ||
lowerMsg.includes('magick') ||
lowerMsg.includes('convert') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format')
) {
throw ApplicationFailure.create({
message: `Image transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
}
}
export interface GenerateSpriteActivityParams {
taskId?: string
assetKey: string
filePath: string
mediaInfo: PrismaJson.MediaInfo
spriteSpec: PrismaJson.SpriteInfo
posterSpec: PrismaJson.PosterInfo
}
export async function generateSpriteActivity(params: GenerateSpriteActivityParams) {
const bucket = process.env.S3_BUCKET || 'shumai'
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Sprite/Poster generation cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, params.spriteSpec.key)
await s3Service.headObject(bucket, params.posterSpec.key)
return { sprite: params.spriteSpec, poster: params.posterSpec }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const spriteFile = path.join(tmpDir, 'sprite.webp')
const posterFile = path.join(tmpDir, 'poster.webp')
try {
if (params.mediaInfo.proxyType === 'pdf') {
const pdfRes = await transcodeService.generatePdfSprite(
params.filePath,
spriteFile,
posterFile,
signal,
)
if (params.mediaInfo && params.mediaInfo.metadata) {
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
params.mediaInfo.frames = pdfRes.pageCount
}
} else {
await transcodeService.generateSprite(
params.filePath,
spriteFile,
posterFile,
params.mediaInfo.duration,
signal,
{
isHdr: params.mediaInfo.metadata?.isHdr,
hdrType: params.mediaInfo.metadata?.hdrType,
colorTransfer: params.mediaInfo.metadata?.colorTransfer,
},
params.mediaInfo.metadata?.videoStreamIndex,
)
}
const spriteBuffer = fs.readFileSync(spriteFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.spriteSpec.key,
spriteBuffer,
spriteBuffer.length,
'image/webp',
)
const posterBuffer = fs.readFileSync(posterFile)
await s3Service.putObject(
bucket,
params.posterSpec.key,
posterBuffer,
posterBuffer.length,
'image/webp',
)
return { sprite: params.spriteSpec, poster: params.posterSpec }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Sprite/Poster generation cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `Sprite/Poster generation failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(spriteFile)) fs.unlinkSync(spriteFile)
if (fs.existsSync(posterFile)) fs.unlinkSync(posterFile)
}
}
export interface GeneratePdfProxyActivityParams {
assetId: string
assetKey: string
filePath: string
mediaType: string
filename: string
}
export async function generatePdfProxyActivity(
params: GeneratePdfProxyActivityParams,
): Promise<{ pdfProxyKey: string; pdfFilePath: string }> {
const isOffice = isOfficeDocument(params.mediaType, params.filename)
const isHtml = isHtmlDocument(params.mediaType, params.filename)
const isMd = isMarkdownDocument(params.mediaType, params.filename)
const isCsv = isCsvDocument(params.mediaType, params.filename)
const isTxt = params.mediaType === 'text/plain' || params.filename.toLowerCase().endsWith('.txt')
if (!isOffice && !isHtml && !isMd && !isCsv && !isTxt) {
return { pdfProxyKey: params.assetKey, pdfFilePath: params.filePath }
}
const bucket = process.env.S3_BUCKET || 'shumai'
const assetDir = getDerivedArtifactDirectory(params.assetKey, params.assetId)
const pdfProxyKey = `${assetDir}/proxy.pdf`
const tmpDir = path.dirname(params.filePath)
const pdfFilePath = path.join(tmpDir, 'proxy.pdf')
try {
const gotenbergAvailable = await gotenbergService.isAvailable()
if (isOffice) {
if (!gotenbergAvailable) {
throw new Error(
'Gotenberg service is required for Office document conversion but is unavailable.',
)
}
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
params.filePath,
params.filename,
)
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else if (isHtml) {
if (!gotenbergAvailable) {
throw new Error(
'Gotenberg service is required for HTML document conversion but is unavailable.',
)
}
const pdfBuffer = await gotenbergService.convertHtmlToPdf(params.filePath, params.filename)
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else if (isCsv) {
if (gotenbergAvailable) {
let isLandscape = false
try {
const content = String(fs.readFileSync(params.filePath, 'utf-8'))
const rows = parseCsvContent(content)
const maxCols = Math.max(...rows.map((r) => r.length), 0)
isLandscape = maxCols > 5
} catch {
// Fallback landscape calculation
}
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
params.filePath,
params.filename,
{ landscape: isLandscape },
)
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else {
await transcodeService.generatePdfFromCsv(params.filePath, pdfFilePath)
}
} else if (isMd) {
if (gotenbergAvailable) {
const pdfBuffer = await gotenbergService.convertMarkdownToPdf(
params.filePath,
params.filename,
)
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else {
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
}
} else if (isTxt) {
if (gotenbergAvailable) {
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
params.filePath,
params.filename,
)
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else {
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
}
}
const stat = fs.statSync(pdfFilePath)
const stream = Bun.file(pdfFilePath).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, pdfProxyKey, stream, stat.size, 'application/pdf')
return { pdfProxyKey, pdfFilePath }
} catch (err) {
const { message } = getErrorDetails(err)
throw ApplicationFailure.create({
message: `Failed to generate PDF proxy: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
}
export async function downloadMediaToTmpActivity(params: {
assetKey: string
}): Promise<{ filePath: string; tmpDir: string }> {
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = transcodeService.createTempDir('transcode-')
const filePath = path.join(tmpDir, path.basename(params.assetKey))
if (!fs.existsSync(tmpDir)) {
fs.mkdirSync(tmpDir, { recursive: true })
}
try {
await s3Service.downloadToFile(bucket, params.assetKey, filePath)
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (code === 'ENOENT' || lowerMsg.includes('enoent') || lowerMsg.includes('nosuchkey')) {
throw ApplicationFailure.create({
message: `Failed to download media to tmp: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
return { filePath, tmpDir }
}
export async function cleanupTmpDirActivity(params: { tmpDir: string }): Promise<void> {
transcodeService.removeDir(params.tmpDir)
}
export interface UpdateAssetMediaActivityParams {
assetId: string
mediaInfo: PrismaJson.MediaInfo
}
export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityParams) {
const bucket = process.env.S3_BUCKET || 'shumai'
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: { storageKey: true },
})
const key = asset?.storageKey?.key
if (!asset || !key || asset.status === AssetStatus.pending_purge) {
throw ApplicationFailure.create({
message: 'Asset not found, has no key, or is being purged',
nonRetryable: true,
})
}
const infoKey = path.join(path.dirname(key), 'info.json')
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
const { count } = await prisma.asset.updateMany({
where: {
id: params.assetId,
status: { not: AssetStatus.pending_purge },
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: { media: params.mediaInfo as any, hasJpegPreview: false },
})
if (count === 0) {
logger.info(
{ assetId: params.assetId },
'[updateAssetMediaActivity] Asset was deleted or pending purge before media update completed',
)
}
}
export async function takeScreenshotsActivity(params: {
taskId?: string
assetKey: string
assetId: string
start: number
end: number
count: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; timestamp: number }>> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Screenshot extraction cancelled',
nonRetryable: true,
})
}
try {
return await transcodeService.takeScreenshots({ ...params, signal })
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Screenshot extraction cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `Screenshot extraction failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export interface ExtractPosterActivityParams {
assetKey: string
posterSpec: PrismaJson.PosterInfo
taskId?: string
signal?: AbortSignal
isHdr?: boolean
hdrType?: HdrType
colorTransfer?: string
streamIndex?: number
}
export async function extractPosterActivity(
params: ExtractPosterActivityParams,
): Promise<{ poster: PrismaJson.PosterInfo }> {
const signal = params.signal || getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Poster extraction cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
try {
await s3Service.headObject(bucket, params.posterSpec.key)
return { poster: params.posterSpec }
} catch {
// Not found, proceed to extract
}
const tmpDir = transcodeService.createTempDir('poster-')
const posterFile = path.join(tmpDir, 'poster.webp')
try {
const inputSource = await s3Service.resolveInput(bucket, params.assetKey)
await transcodeService.generatePoster(inputSource, posterFile, {
isHdr: params.isHdr,
hdrType: params.hdrType,
colorTransfer: params.colorTransfer,
signal,
streamIndex: params.streamIndex,
})
const posterBuffer = fs.readFileSync(posterFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.posterSpec.key,
posterBuffer,
posterBuffer.length,
'image/webp',
)
return { poster: params.posterSpec }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Poster extraction cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('sharp') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `Poster extraction failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
transcodeService.removeDir(tmpDir)
}
}
export async function overlayAnnotationsActivity(params: {
assetKey: string
assetId: string
annotations: PrismaJson.AnnotationList
}): Promise<string> {
try {
return await transcodeService.overlayAnnotations(params)
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `Overlay annotations failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export async function renderPdfPagesActivity(params: {
taskId?: string
assetKey: string
assetId: string
start: number
end: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; page: number }>> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'PDF page rendering cancelled',
nonRetryable: true,
})
}
try {
return await transcodeService.renderPdfPages({ ...params, signal })
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'PDF page rendering cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('pdftoppm') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `PDF page rendering failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export interface CreateEmbeddingTaskIfEnabledParams {
assetId: string
teamId: string | null
projectId: string | null
}
export async function createEmbeddingTaskIfEnabledActivity(
params: CreateEmbeddingTaskIfEnabledParams,
): Promise<void> {
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: {
project: true,
},
})
if (!asset) {
return
}
const isVideo = asset.mediaType?.startsWith('video/')
const isImage = asset.mediaType?.startsWith('image/')
if (!isVideo && !isImage) {
return
}
const teamId = params.teamId || asset.project?.teamId || null
const projectId = params.projectId || asset.projectId
if (!teamId) {
return
}
// Check if there is an active embedding agent for the team
const embeddingAgent = await prisma.agent.findFirst({
where: {
type: 'embedding',
enabled: true,
user: { teamMembers: { some: { teamId } } },
},
})
if (!embeddingAgent) {
return
}
// Check if a pending/processing embedding task already exists for this asset to avoid duplicates
const existing = await prisma.workflowTask.findFirst({
where: {
assetId: params.assetId,
type: WorkflowTaskType.ai_embedding,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
})
if (existing) {
return
}
// Create embedding task
await prisma.workflowTask.create({
data: {
assetId: params.assetId,
type: WorkflowTaskType.ai_embedding,
status: WorkflowTaskStatus.pending,
teamId,
projectId,
payload: {
projectId: projectId ?? '',
agent: { agentId: embeddingAgent.id },
},
},
})
}
export interface CreateAutofillTaskIfEnabledParams {
assetId: string
teamId: string | null
projectId: string | null
}
export async function createAutofillTaskIfEnabledActivity(
params: CreateAutofillTaskIfEnabledParams,
): Promise<void> {
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: {
project: true,
},
})
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack')) {
return
}
const teamId = params.teamId || asset.project?.teamId || null
const projectId = params.projectId || asset.projectId
if (!teamId) {
return
}
// Check if there is an active autofill agent for the team
const autofillAgent = await prisma.agent.findFirst({
where: {
type: 'autofill',
enabled: true,
user: { teamMembers: { some: { teamId } } },
},
})
if (!autofillAgent) {
return
}
// Check if a pending/processing autofill task already exists for this asset to avoid duplicates
const existing = await prisma.workflowTask.findFirst({
where: {
assetId: params.assetId,
type: WorkflowTaskType.ai_metadata_autofill,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
})
if (existing) {
return
}
// Create autofill task
await prisma.workflowTask.create({
data: {
assetId: params.assetId,
type: WorkflowTaskType.ai_metadata_autofill,
status: WorkflowTaskStatus.pending,
teamId,
projectId,
payload: {
projectId: projectId ?? '',
agent: { agentId: autofillAgent.id },
},
},
})
}
export interface TranscodeVideoChunkParams {
assetId: string
assetKey?: string
filePath?: string
startTime: number
endTime: number
}
export async function transcodeVideoChunkActivity(
params: TranscodeVideoChunkParams,
): Promise<{ chunkKey: string }> {
if (!params.filePath && !params.assetKey) {
throw ApplicationFailure.create({
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
nonRetryable: true,
})
}
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
try {
const bucket = process.env.S3_BUCKET || 'shumai'
const inputSource =
params.filePath ||
(params.assetKey ? await s3Service.resolveInput(bucket, params.assetKey) : '')
if (!inputSource) {
throw ApplicationFailure.create({
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
nonRetryable: true,
})
}
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
// Slice video segment using ffmpeg, force 1fps and remove audio for gemini-embedding-2 optimization
// Note: -ss placed before -i for fast input seeking via demuxer Range requests
await execFileAsync('ffmpeg', [
'-y',
'-loglevel',
'warning',
...(isRemote
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
: []),
'-ss',
params.startTime.toString(),
'-i',
inputSource,
'-map',
'0:V:0',
'-t',
(params.endTime - params.startTime).toString(),
'-vf',
'fps=1',
'-c:v',
'libx264',
'-preset',
'ultrafast',
'-crf',
'28',
'-an',
chunkTmp,
])
const chunkData = fs.readFileSync(chunkTmp)
// Upload to S3
const key = `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}-${ulid()}.mp4`
await s3Service.putObject(bucket, key, chunkData, chunkData.length, 'video/mp4')
return { chunkKey: key }
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
throw ApplicationFailure.create({
message: `Failed to transcode video chunk for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
nonRetryable: false,
})
} finally {
if (fs.existsSync(chunkTmp)) {
try {
fs.unlinkSync(chunkTmp)
} catch (e) {
console.error('Failed to cleanup local chunk file:', e)
}
}
}
}
export async function deleteS3ObjectActivity(params: { key: string }): Promise<void> {
const bucket = process.env.S3_BUCKET || 'shumai'
await s3Service.deleteObject(bucket, params.key)
}