import type { WorkflowTask } from '@shumai/db' import '@shumai/db/src/prisma-json-types' import { updateAssetStatusActivity } from '@shumai/workflow-core' import { downloadMediaToTmpActivity, getMediaInfoActivity, transcodeVideoActivity, transcodeHlsActivity, transcodeAudioActivity, transcodeImageActivity, extractPosterActivity, generateSpriteActivity, updateAssetMediaActivity, createEmbeddingTaskIfEnabledActivity, createAutofillTaskIfEnabledActivity, } from '../activities/transcode' import { startTask, fetchAssetWithKey, completeTask, failTask, cleanupTmpDir } from './common' import { getTargetVideoResolutions, getTargetHlsResolutions, resolutionToDimensions, } from './transcode-utils' export async function transcodeVideoWorkflow(task: WorkflowTask): Promise { let tmpDir: string | undefined let mediaProcessed = false try { await startTask(task) await updateAssetStatusActivity({ assetId: task.assetId, status: 'processing', }) const { asset, key } = await fetchAssetWithKey(task.assetId) const spec = task.payload?.transcode || {} const proxyType = asset.mediaType?.startsWith('audio/') ? 'audio' : 'video' const mediaInfo = await getMediaInfoActivity({ assetKey: key, assetId: asset.id, proxyType, mediaType: asset.mediaType || '', }) mediaInfo.original = { key, filesizeInBytes: 0, codec: '', } const isVideo = mediaInfo.proxyType === 'video' if (spec.poster && isVideo) { const lastSlashIndex = key.lastIndexOf('/') const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex) const posterSpec: PrismaJson.PosterInfo = { key: assetDir ? `${assetDir}/poster.webp` : 'poster.webp', } const posterResult = await extractPosterActivity({ assetKey: key, posterSpec, taskId: task.id, isHdr: mediaInfo.metadata?.isHdr, hdrType: mediaInfo.metadata?.hdrType, colorTransfer: mediaInfo.metadata?.colorTransfer, streamIndex: mediaInfo.metadata?.videoStreamIndex, }) mediaInfo.poster = posterResult.poster await updateAssetMediaActivity({ assetId: asset.id, mediaInfo, }) } const download = await downloadMediaToTmpActivity({ assetKey: key, }) const { filePath } = download tmpDir = download.tmpDir // Generate the sprite (and its poster fallback) before the expensive proxy transcodes so the // file list can show a poster/sprite preview while the asset is still processing. if ((spec.sprite || (spec.poster && !mediaInfo.poster)) && isVideo) { const lastSlashIndex = key.lastIndexOf('/') const assetDir = lastSlashIndex === -1 ? '' : key.substring(0, lastSlashIndex) const spriteSpec: PrismaJson.SpriteInfo = { key: assetDir ? `${assetDir}/sprite.webp` : 'sprite.webp', frames: 100, tileX: 10, tileY: 10, } const posterSpec: PrismaJson.PosterInfo = mediaInfo.poster || { key: assetDir ? `${assetDir}/poster.webp` : 'poster.webp', } const spriteResult = await generateSpriteActivity({ taskId: task.id, assetKey: key, filePath, spriteSpec, posterSpec, mediaInfo, }) if (spec.sprite) { mediaInfo.sprite = spriteResult.sprite } if (!mediaInfo.poster) { mediaInfo.poster = spriteResult.poster } await updateAssetMediaActivity({ assetId: asset.id, mediaInfo, }) } const metadata = mediaInfo.metadata if (isVideo && metadata) { const videoResolutions = getTargetVideoResolutions( spec.videoStrategy || 'best_match', metadata.originalWidth, metadata.originalHeight, spec.videoResolutions, ) for (const res of videoResolutions) { const [width, height] = resolutionToDimensions( res, metadata.originalWidth, metadata.originalHeight, ) const isHdr = Boolean(metadata.isHdr) if (res === '180p') { const videoSpec: PrismaJson.VideoTranscode = { resolution: res, width, height, hdr: isHdr, } const videoTranscode = await transcodeVideoActivity({ taskId: task.id, assetKey: key, filePath, videoSpec, duration: metadata.duration, originalFps: metadata.frameRate, hardwareAcceleration: spec.hardwareAcceleration, sourceVideoBitrate: metadata.videoBitRate || metadata.bitRate, threads: spec.threads, sourceIsHdr: metadata.isHdr, sourceHdrType: metadata.hdrType, sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, sourceRotation: metadata.rotation ?? 0, streamIndex: metadata.videoStreamIndex, audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoPreview = videoTranscode continue } const videoSpec: PrismaJson.VideoTranscode = { resolution: res, width, height, hdr: isHdr, } const videoTranscode = await transcodeVideoActivity({ taskId: task.id, assetKey: key, filePath, videoSpec, duration: metadata.duration, originalFps: metadata.frameRate, hardwareAcceleration: spec.hardwareAcceleration, sourceVideoBitrate: metadata.videoBitRate || metadata.bitRate, threads: spec.threads, sourceIsHdr: metadata.isHdr, sourceHdrType: metadata.hdrType, sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, // rotation is undefined when the probed source has none sourceRotation: metadata.rotation ?? 0, streamIndex: metadata.videoStreamIndex, audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoTranscodes.push(videoTranscode) } if (spec.hlsEnabled) { const hlsLadders = getTargetHlsResolutions( spec.hlsResolutions, metadata.originalWidth, metadata.originalHeight, ) if (hlsLadders.length > 0) { const hlsResult = await transcodeHlsActivity({ taskId: task.id, assetKey: key, filePath, ladders: hlsLadders, originalWidth: metadata.originalWidth, originalHeight: metadata.originalHeight, originalFps: metadata.frameRate, duration: metadata.duration, sourceVideoBitrate: metadata.videoBitRate || metadata.bitRate, hardwareAcceleration: spec.hardwareAcceleration, threads: spec.threads, sourceIsHdr: metadata.isHdr, sourceHdrType: metadata.hdrType, sourceColorTransfer: metadata.colorTransfer, sourceColorPrimaries: metadata.colorPrimaries, sourceColorSpace: metadata.colorSpace, sourceRotation: metadata.rotation ?? 0, streamIndex: metadata.videoStreamIndex, audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.isHls = true mediaInfo.hls = hlsResult } } } const isAudio = mediaInfo.proxyType === 'audio' if (isAudio && metadata) { const audioTranscode = await transcodeAudioActivity({ taskId: task.id, assetKey: key, filePath, threads: spec.threads, audioStreamIndex: metadata.audioStreamIndex, }) mediaInfo.videoTranscodes.push(audioTranscode) } if (spec.thumbnail && isVideo) { const thumbTranscode = await transcodeImageActivity({ assetKey: key, filePath, imageSpec: { width: 300, height: 300, quality: 80, format: 'webp', isPreview: true }, }) mediaInfo.thumbnail = thumbTranscode } await updateAssetMediaActivity({ assetId: asset.id, mediaInfo, }) await updateAssetStatusActivity({ assetId: asset.id, status: 'processed', }) mediaProcessed = true await createEmbeddingTaskIfEnabledActivity({ assetId: asset.id, teamId: task.teamId, projectId: task.projectId, }) await createAutofillTaskIfEnabledActivity({ assetId: asset.id, teamId: task.teamId, projectId: task.projectId, }) await completeTask(task.id) } catch (err) { console.error(`transcodeVideoWorkflow failed for task ${task.id}:`, err) await failTask(task.id, err, mediaProcessed ? undefined : task.assetId) throw err } finally { await cleanupTmpDir(tmpDir) } }