mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 10:56:52 +00:00
* fix(transcode): avoid selecting attached picture streams for video transcoding Filter out attached picture streams (disposition.attached_pic) during video probing and stream selection, matching Immich's default and bitrate selection strategy. Forward explicit primary video and audio stream indices to FFmpeg pipelines to prevent picking embedded cover art or thumbnail streams. * fix(transcode): skip poster and sprite generation for audio-only assets and add multi-stream e2e tests * test(workflow): scope pixel format to h264 stream and use yuvj420p for mjpeg cover art
1548 lines
46 KiB
TypeScript
1548 lines
46 KiB
TypeScript
import { AssetStatus, prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
|
import { s3Service } from '@shumai/core/src/s3/s3'
|
|
import {
|
|
transcodeService,
|
|
buildHlsMasterPlaylist,
|
|
calculateEffectiveBitrateBps,
|
|
type HdrType,
|
|
} from '@shumai/core/src/transcode/transcode'
|
|
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
|
import { getDerivedArtifactDirectory, stemFromKey } from '@shumai/core/src/utils/filename'
|
|
import { gotenbergService } from '@shumai/core/src/gotenberg/gotenberg'
|
|
import { parseCsvContent } from '@shumai/core/src/transcode/transcode'
|
|
import { resolutionToDimensions } from '../workflows/transcode-utils'
|
|
import {
|
|
getProxyType,
|
|
isCsvDocument,
|
|
isHtmlDocument,
|
|
isMarkdownDocument,
|
|
isOfficeDocument,
|
|
} from '@shumai/core/src/utils/mime'
|
|
import { logger } from '@shumai/core/src/logger'
|
|
import { ApplicationFailure, Context } from '@temporalio/activity'
|
|
import { getLocalTaskAbortSignal } from '@shumai/workflow-core'
|
|
|
|
function getActivityCancellationSignal(taskId?: string): AbortSignal | undefined {
|
|
try {
|
|
return Context.current().cancellationSignal
|
|
} catch {
|
|
return getLocalTaskAbortSignal(taskId)
|
|
}
|
|
}
|
|
import * as path from 'path'
|
|
import * as fs from 'fs'
|
|
import * as os from 'os'
|
|
import { execFile } from 'child_process'
|
|
import { promisify } from 'util'
|
|
import { ulid } from 'ulid'
|
|
|
|
const execFileAsync = promisify(execFile)
|
|
|
|
export interface GetMediaInfoActivityParams {
|
|
assetId: string
|
|
assetKey: string
|
|
mediaType: string
|
|
}
|
|
|
|
function getErrorDetails(err: unknown): { code?: string; message: string; name?: string } {
|
|
if (err && typeof err === 'object') {
|
|
const record = err as Record<string, unknown>
|
|
return {
|
|
code: typeof record.code === 'string' ? record.code : undefined,
|
|
message:
|
|
typeof record.message === 'string'
|
|
? record.message
|
|
: err instanceof Error
|
|
? err.message
|
|
: String(err),
|
|
name: typeof record.name === 'string' ? record.name : undefined,
|
|
}
|
|
}
|
|
return {
|
|
message: String(err),
|
|
}
|
|
}
|
|
|
|
async function ensureAssetNotPurging(assetKey: string): Promise<void> {
|
|
const sk = await prisma.storageKey.findUnique({
|
|
where: { key: assetKey },
|
|
include: { assets: { select: { id: true, status: true } } },
|
|
})
|
|
if (
|
|
sk &&
|
|
(sk.status === 'purging' ||
|
|
sk.assets.length === 0 ||
|
|
sk.assets.every((a) => a.status === AssetStatus.pending_purge))
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Asset or storage key has been purged or is pending purge',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
}
|
|
|
|
export async function getMediaInfoActivity(params: {
|
|
filePath: string
|
|
assetId: string
|
|
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
|
|
mediaType?: string
|
|
}): Promise<PrismaJson.MediaInfo> {
|
|
try {
|
|
const proxyType =
|
|
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
|
|
const isVideo = proxyType === 'video'
|
|
const isImage = proxyType === 'image'
|
|
const isAudio = proxyType === 'audio'
|
|
const isPdf = proxyType === 'pdf'
|
|
|
|
const fileType = isVideo
|
|
? 'video'
|
|
: isAudio
|
|
? 'audio'
|
|
: isImage
|
|
? 'image'
|
|
: isPdf
|
|
? 'document'
|
|
: 'file'
|
|
|
|
const mediaInfo: PrismaJson.MediaInfo = {
|
|
duration: 0,
|
|
filesize: 0,
|
|
frames: 0,
|
|
proxyType,
|
|
imageTranscodes: [],
|
|
videoTranscodes: [],
|
|
videoPreview: { width: 0, height: 0 },
|
|
finishedAt: new Date().toISOString(),
|
|
metadata: null,
|
|
original: {
|
|
key: '', // Will be filled by caller or updated later
|
|
filesizeInBytes: 0,
|
|
codec: '',
|
|
},
|
|
}
|
|
|
|
const metadataUpdates: { key: string; value: string | number }[] = [
|
|
{ key: 'file_type', value: fileType },
|
|
]
|
|
if (proxyType) {
|
|
metadataUpdates.push({ key: 'proxy_type', value: proxyType })
|
|
}
|
|
|
|
if (isVideo) {
|
|
let isDowngradedToAudio = false
|
|
let videoInfo: Awaited<ReturnType<typeof transcodeService.getVideoInfo>> | null = null
|
|
try {
|
|
videoInfo = await transcodeService.getVideoInfo(params.filePath)
|
|
} catch (err) {
|
|
if (err instanceof Error && err.message.includes('No video stream found')) {
|
|
try {
|
|
const audioInfo = await transcodeService.getAudioInfo(params.filePath)
|
|
isDowngradedToAudio = true
|
|
mediaInfo.proxyType = 'audio'
|
|
mediaInfo.duration = audioInfo.duration
|
|
mediaInfo.metadata = {
|
|
originalWidth: 0,
|
|
originalHeight: 0,
|
|
duration: audioInfo.duration,
|
|
bitRate: audioInfo.bitRate,
|
|
frameRate: 0,
|
|
totalFrames: 0,
|
|
startTimecode: '00:00:00:00',
|
|
hasAudio: true,
|
|
videoCodec: undefined,
|
|
audioCodec: audioInfo.audioCodec,
|
|
audioChannels: audioInfo.audioChannels,
|
|
audioSampleRate: audioInfo.audioSampleRate,
|
|
audioBitDepth: audioInfo.audioBitDepth,
|
|
audioStreamIndex: audioInfo.audioStreamIndex,
|
|
format: {},
|
|
}
|
|
const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type')
|
|
if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio'
|
|
const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type')
|
|
if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio'
|
|
metadataUpdates.push(
|
|
{ key: 'duration', value: audioInfo.duration },
|
|
{ key: 'bitRate', value: audioInfo.bitRate / 1000 },
|
|
)
|
|
if (audioInfo.audioCodec)
|
|
metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec })
|
|
if (audioInfo.audioChannels !== undefined)
|
|
metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels })
|
|
if (audioInfo.audioSampleRate !== undefined)
|
|
metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate })
|
|
if (audioInfo.audioBitDepth !== undefined)
|
|
metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth })
|
|
} catch {
|
|
throw err
|
|
}
|
|
} else {
|
|
throw err
|
|
}
|
|
}
|
|
|
|
if (!isDowngradedToAudio && videoInfo) {
|
|
const info = videoInfo
|
|
mediaInfo.duration = info.duration
|
|
mediaInfo.frames = info.totalFrames
|
|
mediaInfo.metadata = {
|
|
originalWidth: info.originalWidth,
|
|
originalHeight: info.originalHeight,
|
|
duration: info.duration,
|
|
bitRate: info.bitRate,
|
|
videoBitRate: info.videoBitRate,
|
|
frameRate: info.frameRate,
|
|
totalFrames: info.totalFrames,
|
|
startTimecode: info.startTimecode || '00:00:00:00',
|
|
hasAudio: info.hasAudio,
|
|
videoCodec: info.videoCodec,
|
|
audioCodec: info.audioCodec,
|
|
audioChannels: info.audioChannels,
|
|
audioSampleRate: info.audioSampleRate,
|
|
audioBitDepth: info.audioBitDepth,
|
|
colorTransfer: info.colorTransfer,
|
|
colorPrimaries: info.colorPrimaries,
|
|
colorSpace: info.colorSpace,
|
|
isHdr: info.isHdr,
|
|
hdrType: info.hdrType,
|
|
dvProfile: info.dvProfile,
|
|
rotation: info.rotation,
|
|
videoStreamIndex: info.videoStreamIndex,
|
|
audioStreamIndex: info.audioStreamIndex,
|
|
format: {},
|
|
}
|
|
metadataUpdates.push(
|
|
{ key: 'resolution_width', value: info.originalWidth },
|
|
{ key: 'resolution_height', value: info.originalHeight },
|
|
{ key: 'duration', value: info.duration },
|
|
{ key: 'bitRate', value: info.bitRate / 1000 },
|
|
{ key: 'frame_rate', value: info.frameRate },
|
|
)
|
|
if (info.rotation !== undefined)
|
|
metadataUpdates.push({ key: 'rotation', value: info.rotation })
|
|
if (info.isHdr !== undefined)
|
|
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
|
|
if (info.colorTransfer)
|
|
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
|
|
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
|
|
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
|
|
if (info.audioChannels !== undefined)
|
|
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
|
|
if (info.audioSampleRate !== undefined)
|
|
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
|
|
if (info.audioBitDepth !== undefined)
|
|
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
|
|
}
|
|
} else if (isAudio) {
|
|
const info = await transcodeService.getAudioInfo(params.filePath)
|
|
mediaInfo.duration = info.duration
|
|
mediaInfo.metadata = {
|
|
originalWidth: 0,
|
|
originalHeight: 0,
|
|
duration: info.duration,
|
|
bitRate: info.bitRate,
|
|
frameRate: 0,
|
|
totalFrames: 0,
|
|
startTimecode: '00:00:00:00',
|
|
hasAudio: true,
|
|
videoCodec: undefined,
|
|
audioCodec: info.audioCodec,
|
|
audioChannels: info.audioChannels,
|
|
audioSampleRate: info.audioSampleRate,
|
|
audioBitDepth: info.audioBitDepth,
|
|
audioStreamIndex: info.audioStreamIndex,
|
|
format: {},
|
|
}
|
|
metadataUpdates.push(
|
|
{ key: 'duration', value: info.duration },
|
|
{ key: 'bitRate', value: info.bitRate / 1000 },
|
|
)
|
|
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
|
|
if (info.audioChannels !== undefined)
|
|
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
|
|
if (info.audioSampleRate !== undefined)
|
|
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
|
|
if (info.audioBitDepth !== undefined)
|
|
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
|
|
} else if (isImage) {
|
|
const info = await transcodeService.getImageInfo(params.filePath)
|
|
mediaInfo.metadata = {
|
|
originalWidth: info.originalWidth,
|
|
originalHeight: info.originalHeight,
|
|
duration: 0,
|
|
bitRate: 0,
|
|
frameRate: 0,
|
|
totalFrames: 0,
|
|
startTimecode: '00:00:00:00',
|
|
hasAudio: false,
|
|
format: {},
|
|
}
|
|
metadataUpdates.push(
|
|
{ key: 'resolution_width', value: info.originalWidth },
|
|
{ key: 'resolution_height', value: info.originalHeight },
|
|
)
|
|
} else if (isPdf) {
|
|
const info = await transcodeService.getPdfInfo(params.filePath)
|
|
mediaInfo.frames = info.totalFrames
|
|
mediaInfo.metadata = {
|
|
originalWidth: info.originalWidth,
|
|
originalHeight: info.originalHeight,
|
|
duration: 0,
|
|
bitRate: 0,
|
|
frameRate: 0,
|
|
totalFrames: info.totalFrames,
|
|
startTimecode: '00:00:00:00',
|
|
hasAudio: false,
|
|
format: {},
|
|
}
|
|
metadataUpdates.push(
|
|
{ key: 'resolution_width', value: info.originalWidth },
|
|
{ key: 'resolution_height', value: info.originalHeight },
|
|
{ key: 'total_pages', value: info.totalFrames },
|
|
)
|
|
}
|
|
|
|
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
|
|
|
|
return mediaInfo
|
|
} catch (err) {
|
|
const { name, message } = getErrorDetails(err)
|
|
if (name?.includes('Prisma') || message.includes('Prisma')) {
|
|
throw err
|
|
}
|
|
throw ApplicationFailure.create({
|
|
message: `Failed to get media info: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
}
|
|
|
|
export interface VideoActivityParams {
|
|
taskId?: string
|
|
assetKey: string
|
|
filePath: string
|
|
videoSpec: PrismaJson.VideoTranscode
|
|
duration: number
|
|
originalFps: number
|
|
hardwareAcceleration?: PrismaJson.HardwareAcceleration
|
|
sourceVideoBitrate?: number
|
|
threads?: number
|
|
sourceIsHdr?: boolean
|
|
sourceHdrType?: HdrType
|
|
sourceColorTransfer?: string
|
|
sourceColorPrimaries?: string
|
|
sourceColorSpace?: string
|
|
streamIndex?: number
|
|
audioStreamIndex?: number
|
|
}
|
|
|
|
export async function transcodeVideoActivity(
|
|
params: VideoActivityParams,
|
|
): Promise<PrismaJson.VideoTranscode> {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const stem = stemFromKey(params.assetKey)
|
|
const res = params.videoSpec.resolution || `${params.videoSpec.height}p`
|
|
const suffix = params.videoSpec.hdr ? `${res}-hdr` : res
|
|
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-${suffix}.mp4`)
|
|
|
|
const signal = getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Video transcoding cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
try {
|
|
await s3Service.headObject(bucket, key)
|
|
return { ...params.videoSpec, key }
|
|
} catch {
|
|
// Not found
|
|
}
|
|
|
|
const tmpDir = path.dirname(params.filePath)
|
|
const outputFile = path.join(tmpDir, `${stem}-${suffix}.mp4`)
|
|
|
|
try {
|
|
let targetFps: number | string = params.originalFps || 30
|
|
let disableAudio = false
|
|
|
|
if (params.videoSpec.resolution === '180p') {
|
|
disableAudio = true
|
|
targetFps = 24.0
|
|
if (params.originalFps > 0 && params.originalFps < 24.0) {
|
|
targetFps = params.originalFps
|
|
}
|
|
if (params.duration * (targetFps as number) > 1600) {
|
|
targetFps = `160000/${Math.floor(params.duration * 100)}`
|
|
}
|
|
}
|
|
|
|
await transcodeService.transcodeVideo({
|
|
inputFile: params.filePath,
|
|
outputFile,
|
|
width: params.videoSpec.width,
|
|
height: params.videoSpec.height,
|
|
frameRate: targetFps || undefined,
|
|
disableAudio,
|
|
hardwareAcceleration: params.hardwareAcceleration,
|
|
sourceVideoBitrate: params.sourceVideoBitrate,
|
|
threads: params.threads,
|
|
signal,
|
|
hdr: params.videoSpec.hdr,
|
|
sourceIsHdr: params.sourceIsHdr,
|
|
sourceHdrType: params.sourceHdrType,
|
|
sourceColorTransfer: params.sourceColorTransfer,
|
|
sourceColorPrimaries: params.sourceColorPrimaries,
|
|
sourceColorSpace: params.sourceColorSpace,
|
|
streamIndex: params.streamIndex,
|
|
audioStreamIndex: params.audioStreamIndex,
|
|
})
|
|
|
|
const stat = fs.statSync(outputFile)
|
|
const stream = Bun.file(outputFile).stream()
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
|
|
|
|
return { ...params.videoSpec, key }
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Video transcoding cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('ffprobe') ||
|
|
lowerMsg.includes('spawn') ||
|
|
lowerMsg.includes('format') ||
|
|
lowerMsg.includes('no video stream found') ||
|
|
lowerMsg.includes('zscale') ||
|
|
lowerMsg.includes('tonemap')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Video transcoding failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
} finally {
|
|
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
|
|
}
|
|
}
|
|
|
|
export interface HlsActivityParams {
|
|
taskId: string
|
|
assetKey: string
|
|
filePath: string
|
|
ladders: PrismaJson.HlsResolutionLadder[]
|
|
originalWidth: number
|
|
originalHeight: number
|
|
duration: number
|
|
originalFps: number
|
|
hardwareAcceleration?: 'off' | 'auto'
|
|
sourceVideoBitrate?: number
|
|
threads?: number
|
|
sourceIsHdr?: boolean
|
|
sourceHdrType?: HdrType
|
|
sourceColorTransfer?: string
|
|
sourceColorPrimaries?: string
|
|
sourceColorSpace?: string
|
|
streamIndex?: number
|
|
audioStreamIndex?: number
|
|
}
|
|
|
|
export async function transcodeHlsActivity(params: HlsActivityParams): Promise<PrismaJson.HlsInfo> {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const assetDir = path.posix.dirname(params.assetKey)
|
|
const hlsDirKey = path.posix.join(assetDir, 'hls')
|
|
const masterKey = path.posix.join(hlsDirKey, 'master.m3u8')
|
|
|
|
const signal = getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'HLS transcoding cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
const hlsTmpDir = path.join(path.dirname(params.filePath), `hls-${ulid()}`)
|
|
|
|
try {
|
|
const renditionSpecs: Array<{
|
|
resolution: string
|
|
width: number
|
|
height: number
|
|
bitrateBps: number
|
|
isHdr?: boolean
|
|
}> = []
|
|
|
|
const mediaResolutions: PrismaJson.VideoTranscode[] = []
|
|
|
|
for (const ladder of params.ladders) {
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'HLS transcoding cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
const [width, height] = resolutionToDimensions(
|
|
ladder,
|
|
params.originalWidth,
|
|
params.originalHeight,
|
|
)
|
|
const targetBps = calculateEffectiveBitrateBps(
|
|
height,
|
|
width,
|
|
params.sourceVideoBitrate,
|
|
params.originalFps,
|
|
)
|
|
const ladderDir = path.join(hlsTmpDir, ladder)
|
|
|
|
await transcodeService.transcodeHlsRendition({
|
|
inputFile: params.filePath,
|
|
outputDir: ladderDir,
|
|
width,
|
|
height,
|
|
frameRate: params.originalFps,
|
|
hardwareAcceleration: params.hardwareAcceleration,
|
|
sourceVideoBitrate: params.sourceVideoBitrate,
|
|
threads: params.threads,
|
|
signal,
|
|
hdr: Boolean(params.sourceIsHdr),
|
|
sourceIsHdr: params.sourceIsHdr,
|
|
sourceHdrType: params.sourceHdrType,
|
|
sourceColorTransfer: params.sourceColorTransfer,
|
|
sourceColorPrimaries: params.sourceColorPrimaries,
|
|
sourceColorSpace: params.sourceColorSpace,
|
|
streamIndex: params.streamIndex,
|
|
audioStreamIndex: params.audioStreamIndex,
|
|
})
|
|
|
|
renditionSpecs.push({
|
|
resolution: ladder,
|
|
width,
|
|
height,
|
|
bitrateBps: targetBps,
|
|
isHdr: params.sourceIsHdr,
|
|
})
|
|
|
|
mediaResolutions.push({
|
|
resolution: ladder,
|
|
width,
|
|
height,
|
|
hdr: Boolean(params.sourceIsHdr),
|
|
})
|
|
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
const files = fs.readdirSync(ladderDir)
|
|
for (const file of files) {
|
|
const localFilePath = path.join(ladderDir, file)
|
|
const stat = fs.statSync(localFilePath)
|
|
const s3Key = path.posix.join(hlsDirKey, ladder, file)
|
|
let contentType = 'application/octet-stream'
|
|
if (file.endsWith('.m3u8')) {
|
|
contentType = 'application/vnd.apple.mpegurl'
|
|
} else if (file.endsWith('.mp4')) {
|
|
contentType = 'video/mp4'
|
|
} else if (file.endsWith('.m4s')) {
|
|
contentType = 'video/iso.segment'
|
|
}
|
|
|
|
const stream = Bun.file(localFilePath).stream()
|
|
await s3Service.putObject(bucket, s3Key, stream, stat.size, contentType)
|
|
}
|
|
}
|
|
|
|
const masterContent = buildHlsMasterPlaylist(renditionSpecs)
|
|
const masterBuffer = Buffer.from(masterContent, 'utf-8')
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(
|
|
bucket,
|
|
masterKey,
|
|
masterBuffer,
|
|
masterBuffer.length,
|
|
'application/vnd.apple.mpegurl',
|
|
)
|
|
|
|
return {
|
|
key: masterKey,
|
|
resolutions: mediaResolutions,
|
|
}
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'HLS transcoding cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('ffprobe') ||
|
|
lowerMsg.includes('spawn') ||
|
|
lowerMsg.includes('format') ||
|
|
lowerMsg.includes('no video stream found') ||
|
|
lowerMsg.includes('zscale') ||
|
|
lowerMsg.includes('tonemap')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `HLS transcoding failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
} finally {
|
|
if (fs.existsSync(hlsTmpDir)) {
|
|
try {
|
|
fs.rmSync(hlsTmpDir, { recursive: true, force: true })
|
|
} catch {
|
|
// ignore cleanup error
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export interface AudioActivityParams {
|
|
taskId?: string
|
|
assetKey: string
|
|
filePath: string
|
|
threads?: number
|
|
audioStreamIndex?: number
|
|
}
|
|
|
|
export async function transcodeAudioActivity(
|
|
params: AudioActivityParams,
|
|
): Promise<PrismaJson.VideoTranscode> {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const stem = stemFromKey(params.assetKey)
|
|
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
|
|
|
|
const signal = getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Audio transcoding cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
try {
|
|
await s3Service.headObject(bucket, key)
|
|
return { width: 0, height: 0, key }
|
|
} catch {
|
|
// Not found
|
|
}
|
|
|
|
const tmpDir = path.dirname(params.filePath)
|
|
const outputFile = path.join(tmpDir, `${stem}-audio-proxy.mp4`)
|
|
|
|
try {
|
|
await transcodeService.transcodeAudio({
|
|
inputFile: params.filePath,
|
|
outputFile,
|
|
bitrate: '128k',
|
|
threads: params.threads,
|
|
signal,
|
|
audioStreamIndex: params.audioStreamIndex,
|
|
})
|
|
|
|
const stat = fs.statSync(outputFile)
|
|
const stream = Bun.file(outputFile).stream()
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
|
|
|
|
return { width: 0, height: 0, key }
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Audio transcoding cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('ffprobe') ||
|
|
lowerMsg.includes('spawn') ||
|
|
lowerMsg.includes('format') ||
|
|
lowerMsg.includes('no video stream found')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Audio transcoding failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
} finally {
|
|
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
|
|
}
|
|
}
|
|
|
|
export interface ImageActivityParams {
|
|
assetKey: string
|
|
filePath: string
|
|
imageSpec: PrismaJson.ImageTranscode
|
|
}
|
|
|
|
export async function transcodeImageActivity(
|
|
params: ImageActivityParams,
|
|
): Promise<PrismaJson.ImageTranscode> {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const stem = stemFromKey(params.assetKey)
|
|
const key = path.posix.join(
|
|
path.posix.dirname(params.assetKey),
|
|
`${stem}-${params.imageSpec.width}p.webp`,
|
|
)
|
|
|
|
try {
|
|
await s3Service.headObject(bucket, key)
|
|
return { ...params.imageSpec, key }
|
|
} catch {
|
|
// Not found
|
|
}
|
|
|
|
const tmpDir = path.dirname(params.filePath)
|
|
const outputFile = path.join(tmpDir, `${stem}-${params.imageSpec.width}p.webp`)
|
|
|
|
try {
|
|
await transcodeService.transcodeImage(
|
|
params.filePath,
|
|
outputFile,
|
|
params.imageSpec.width,
|
|
params.imageSpec.quality,
|
|
{
|
|
height: params.imageSpec.height || null,
|
|
isPreview: params.imageSpec.isPreview ?? false,
|
|
},
|
|
)
|
|
const buffer = fs.readFileSync(outputFile)
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
|
|
|
|
return { ...params.imageSpec, key }
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('sharp') ||
|
|
lowerMsg.includes('imagemagick') ||
|
|
lowerMsg.includes('magick') ||
|
|
lowerMsg.includes('convert') ||
|
|
lowerMsg.includes('spawn') ||
|
|
lowerMsg.includes('format')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Image transcoding failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
} finally {
|
|
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
|
|
}
|
|
}
|
|
|
|
export interface GenerateSpriteActivityParams {
|
|
taskId?: string
|
|
assetKey: string
|
|
filePath: string
|
|
mediaInfo: PrismaJson.MediaInfo
|
|
spriteSpec: PrismaJson.SpriteInfo
|
|
posterSpec: PrismaJson.PosterInfo
|
|
}
|
|
|
|
export async function generateSpriteActivity(params: GenerateSpriteActivityParams) {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const signal = getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Sprite/Poster generation cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
try {
|
|
await s3Service.headObject(bucket, params.spriteSpec.key)
|
|
await s3Service.headObject(bucket, params.posterSpec.key)
|
|
return { sprite: params.spriteSpec, poster: params.posterSpec }
|
|
} catch {
|
|
// Not found
|
|
}
|
|
|
|
const tmpDir = path.dirname(params.filePath)
|
|
const spriteFile = path.join(tmpDir, 'sprite.webp')
|
|
const posterFile = path.join(tmpDir, 'poster.webp')
|
|
|
|
try {
|
|
if (params.mediaInfo.proxyType === 'pdf') {
|
|
const pdfRes = await transcodeService.generatePdfSprite(
|
|
params.filePath,
|
|
spriteFile,
|
|
posterFile,
|
|
signal,
|
|
)
|
|
if (params.mediaInfo && params.mediaInfo.metadata) {
|
|
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
|
|
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
|
|
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
|
|
params.mediaInfo.frames = pdfRes.pageCount
|
|
}
|
|
} else {
|
|
await transcodeService.generateSprite(
|
|
params.filePath,
|
|
spriteFile,
|
|
posterFile,
|
|
params.mediaInfo.duration,
|
|
signal,
|
|
{
|
|
isHdr: params.mediaInfo.metadata?.isHdr,
|
|
hdrType: params.mediaInfo.metadata?.hdrType,
|
|
colorTransfer: params.mediaInfo.metadata?.colorTransfer,
|
|
},
|
|
params.mediaInfo.metadata?.videoStreamIndex,
|
|
)
|
|
}
|
|
|
|
const spriteBuffer = fs.readFileSync(spriteFile)
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(
|
|
bucket,
|
|
params.spriteSpec.key,
|
|
spriteBuffer,
|
|
spriteBuffer.length,
|
|
'image/webp',
|
|
)
|
|
|
|
const posterBuffer = fs.readFileSync(posterFile)
|
|
await s3Service.putObject(
|
|
bucket,
|
|
params.posterSpec.key,
|
|
posterBuffer,
|
|
posterBuffer.length,
|
|
'image/webp',
|
|
)
|
|
|
|
return { sprite: params.spriteSpec, poster: params.posterSpec }
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Sprite/Poster generation cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('ffprobe') ||
|
|
lowerMsg.includes('spawn') ||
|
|
lowerMsg.includes('format') ||
|
|
lowerMsg.includes('zscale') ||
|
|
lowerMsg.includes('tonemap')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Sprite/Poster generation failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
} finally {
|
|
if (fs.existsSync(spriteFile)) fs.unlinkSync(spriteFile)
|
|
if (fs.existsSync(posterFile)) fs.unlinkSync(posterFile)
|
|
}
|
|
}
|
|
|
|
export interface GeneratePdfProxyActivityParams {
|
|
assetId: string
|
|
assetKey: string
|
|
filePath: string
|
|
mediaType: string
|
|
filename: string
|
|
}
|
|
|
|
export async function generatePdfProxyActivity(
|
|
params: GeneratePdfProxyActivityParams,
|
|
): Promise<{ pdfProxyKey: string; pdfFilePath: string }> {
|
|
const isOffice = isOfficeDocument(params.mediaType, params.filename)
|
|
const isHtml = isHtmlDocument(params.mediaType, params.filename)
|
|
const isMd = isMarkdownDocument(params.mediaType, params.filename)
|
|
const isCsv = isCsvDocument(params.mediaType, params.filename)
|
|
const isTxt = params.mediaType === 'text/plain' || params.filename.toLowerCase().endsWith('.txt')
|
|
|
|
if (!isOffice && !isHtml && !isMd && !isCsv && !isTxt) {
|
|
return { pdfProxyKey: params.assetKey, pdfFilePath: params.filePath }
|
|
}
|
|
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const assetDir = getDerivedArtifactDirectory(params.assetKey, params.assetId)
|
|
const pdfProxyKey = `${assetDir}/proxy.pdf`
|
|
const tmpDir = path.dirname(params.filePath)
|
|
const pdfFilePath = path.join(tmpDir, 'proxy.pdf')
|
|
|
|
try {
|
|
const gotenbergAvailable = await gotenbergService.isAvailable()
|
|
|
|
if (isOffice) {
|
|
if (!gotenbergAvailable) {
|
|
throw new Error(
|
|
'Gotenberg service is required for Office document conversion but is unavailable.',
|
|
)
|
|
}
|
|
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
|
|
params.filePath,
|
|
params.filename,
|
|
)
|
|
fs.writeFileSync(pdfFilePath, pdfBuffer)
|
|
} else if (isHtml) {
|
|
if (!gotenbergAvailable) {
|
|
throw new Error(
|
|
'Gotenberg service is required for HTML document conversion but is unavailable.',
|
|
)
|
|
}
|
|
const pdfBuffer = await gotenbergService.convertHtmlToPdf(params.filePath, params.filename)
|
|
fs.writeFileSync(pdfFilePath, pdfBuffer)
|
|
} else if (isCsv) {
|
|
if (gotenbergAvailable) {
|
|
let isLandscape = false
|
|
try {
|
|
const content = String(fs.readFileSync(params.filePath, 'utf-8'))
|
|
const rows = parseCsvContent(content)
|
|
const maxCols = Math.max(...rows.map((r) => r.length), 0)
|
|
isLandscape = maxCols > 5
|
|
} catch {
|
|
// Fallback landscape calculation
|
|
}
|
|
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
|
|
params.filePath,
|
|
params.filename,
|
|
{ landscape: isLandscape },
|
|
)
|
|
fs.writeFileSync(pdfFilePath, pdfBuffer)
|
|
} else {
|
|
await transcodeService.generatePdfFromCsv(params.filePath, pdfFilePath)
|
|
}
|
|
} else if (isMd) {
|
|
if (gotenbergAvailable) {
|
|
const pdfBuffer = await gotenbergService.convertMarkdownToPdf(
|
|
params.filePath,
|
|
params.filename,
|
|
)
|
|
fs.writeFileSync(pdfFilePath, pdfBuffer)
|
|
} else {
|
|
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
|
|
}
|
|
} else if (isTxt) {
|
|
if (gotenbergAvailable) {
|
|
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
|
|
params.filePath,
|
|
params.filename,
|
|
)
|
|
fs.writeFileSync(pdfFilePath, pdfBuffer)
|
|
} else {
|
|
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
|
|
}
|
|
}
|
|
|
|
const stat = fs.statSync(pdfFilePath)
|
|
const stream = Bun.file(pdfFilePath).stream()
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(bucket, pdfProxyKey, stream, stat.size, 'application/pdf')
|
|
|
|
return { pdfProxyKey, pdfFilePath }
|
|
} catch (err) {
|
|
const { message } = getErrorDetails(err)
|
|
throw ApplicationFailure.create({
|
|
message: `Failed to generate PDF proxy: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
}
|
|
|
|
export async function downloadMediaToTmpActivity(params: {
|
|
assetKey: string
|
|
}): Promise<{ filePath: string; tmpDir: string }> {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const tmpDir = transcodeService.createTempDir('transcode-')
|
|
const filePath = path.join(tmpDir, path.basename(params.assetKey))
|
|
|
|
if (!fs.existsSync(tmpDir)) {
|
|
fs.mkdirSync(tmpDir, { recursive: true })
|
|
}
|
|
|
|
try {
|
|
await s3Service.downloadToFile(bucket, params.assetKey, filePath)
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
if (code === 'ENOENT' || lowerMsg.includes('enoent') || lowerMsg.includes('nosuchkey')) {
|
|
throw ApplicationFailure.create({
|
|
message: `Failed to download media to tmp: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
}
|
|
|
|
return { filePath, tmpDir }
|
|
}
|
|
|
|
export async function cleanupTmpDirActivity(params: { tmpDir: string }): Promise<void> {
|
|
transcodeService.removeDir(params.tmpDir)
|
|
}
|
|
|
|
export interface UpdateAssetMediaActivityParams {
|
|
assetId: string
|
|
mediaInfo: PrismaJson.MediaInfo
|
|
}
|
|
|
|
export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityParams) {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const asset = await prisma.asset.findUnique({
|
|
where: { id: params.assetId },
|
|
include: { storageKey: true },
|
|
})
|
|
const key = asset?.storageKey?.key
|
|
if (!asset || !key || asset.status === AssetStatus.pending_purge) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Asset not found, has no key, or is being purged',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
const infoKey = path.join(path.dirname(key), 'info.json')
|
|
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
|
|
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
|
|
|
|
const { count } = await prisma.asset.updateMany({
|
|
where: {
|
|
id: params.assetId,
|
|
status: { not: AssetStatus.pending_purge },
|
|
},
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
data: { media: params.mediaInfo as any, hasJpegPreview: false },
|
|
})
|
|
if (count === 0) {
|
|
logger.info(
|
|
{ assetId: params.assetId },
|
|
'[updateAssetMediaActivity] Asset was deleted or pending purge before media update completed',
|
|
)
|
|
}
|
|
}
|
|
|
|
export async function takeScreenshotsActivity(params: {
|
|
taskId?: string
|
|
assetKey: string
|
|
assetId: string
|
|
start: number
|
|
end: number
|
|
count: number
|
|
commentTimestamp?: number | null
|
|
annotations?: PrismaJson.AnnotationList | null
|
|
}): Promise<Array<{ key: string; timestamp: number }>> {
|
|
const signal = getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Screenshot extraction cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
try {
|
|
return await transcodeService.takeScreenshots({ ...params, signal })
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Screenshot extraction cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('nosuchkey') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('sharp')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Screenshot extraction failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
}
|
|
}
|
|
|
|
export interface ExtractPosterActivityParams {
|
|
assetKey: string
|
|
posterSpec: PrismaJson.PosterInfo
|
|
taskId?: string
|
|
signal?: AbortSignal
|
|
isHdr?: boolean
|
|
hdrType?: HdrType
|
|
colorTransfer?: string
|
|
streamIndex?: number
|
|
}
|
|
|
|
export async function extractPosterActivity(
|
|
params: ExtractPosterActivityParams,
|
|
): Promise<{ poster: PrismaJson.PosterInfo }> {
|
|
const signal = params.signal || getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Poster extraction cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
try {
|
|
await s3Service.headObject(bucket, params.posterSpec.key)
|
|
return { poster: params.posterSpec }
|
|
} catch {
|
|
// Not found, proceed to extract
|
|
}
|
|
|
|
const tmpDir = transcodeService.createTempDir('poster-')
|
|
const posterFile = path.join(tmpDir, 'poster.webp')
|
|
|
|
try {
|
|
const inputSource = await s3Service.resolveInput(bucket, params.assetKey)
|
|
await transcodeService.generatePoster(inputSource, posterFile, {
|
|
isHdr: params.isHdr,
|
|
hdrType: params.hdrType,
|
|
colorTransfer: params.colorTransfer,
|
|
signal,
|
|
streamIndex: params.streamIndex,
|
|
})
|
|
|
|
const posterBuffer = fs.readFileSync(posterFile)
|
|
await ensureAssetNotPurging(params.assetKey)
|
|
await s3Service.putObject(
|
|
bucket,
|
|
params.posterSpec.key,
|
|
posterBuffer,
|
|
posterBuffer.length,
|
|
'image/webp',
|
|
)
|
|
|
|
return { poster: params.posterSpec }
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Poster extraction cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('nosuchkey') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('sharp') ||
|
|
lowerMsg.includes('zscale') ||
|
|
lowerMsg.includes('tonemap')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Poster extraction failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
} finally {
|
|
transcodeService.removeDir(tmpDir)
|
|
}
|
|
}
|
|
|
|
export async function overlayAnnotationsActivity(params: {
|
|
assetKey: string
|
|
assetId: string
|
|
annotations: PrismaJson.AnnotationList
|
|
}): Promise<string> {
|
|
try {
|
|
return await transcodeService.overlayAnnotations(params)
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('nosuchkey') ||
|
|
lowerMsg.includes('ffmpeg') ||
|
|
lowerMsg.includes('sharp')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `Overlay annotations failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
}
|
|
}
|
|
|
|
export async function renderPdfPagesActivity(params: {
|
|
taskId?: string
|
|
assetKey: string
|
|
assetId: string
|
|
start: number
|
|
end: number
|
|
commentTimestamp?: number | null
|
|
annotations?: PrismaJson.AnnotationList | null
|
|
}): Promise<Array<{ key: string; page: number }>> {
|
|
const signal = getActivityCancellationSignal(params.taskId)
|
|
if (signal?.aborted) {
|
|
throw ApplicationFailure.create({
|
|
message: 'PDF page rendering cancelled',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
try {
|
|
return await transcodeService.renderPdfPages({ ...params, signal })
|
|
} catch (err) {
|
|
const { code, message } = getErrorDetails(err)
|
|
const lowerMsg = message.toLowerCase()
|
|
|
|
if (
|
|
signal?.aborted ||
|
|
code === 'ABORT_ERR' ||
|
|
message.includes('aborted') ||
|
|
lowerMsg.includes('abort')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: 'PDF page rendering cancelled',
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
|
|
if (
|
|
code === 'ENOENT' ||
|
|
lowerMsg.includes('enoent') ||
|
|
lowerMsg.includes('nosuchkey') ||
|
|
lowerMsg.includes('pdftoppm') ||
|
|
lowerMsg.includes('sharp')
|
|
) {
|
|
throw ApplicationFailure.create({
|
|
message: `PDF page rendering failed: ${message}`,
|
|
nonRetryable: true,
|
|
cause: err instanceof Error ? err : undefined,
|
|
})
|
|
}
|
|
throw err
|
|
}
|
|
}
|
|
|
|
export interface CreateEmbeddingTaskIfEnabledParams {
|
|
assetId: string
|
|
teamId: string | null
|
|
projectId: string | null
|
|
}
|
|
|
|
export async function createEmbeddingTaskIfEnabledActivity(
|
|
params: CreateEmbeddingTaskIfEnabledParams,
|
|
): Promise<void> {
|
|
const asset = await prisma.asset.findUnique({
|
|
where: { id: params.assetId },
|
|
include: {
|
|
project: true,
|
|
},
|
|
})
|
|
if (!asset) {
|
|
return
|
|
}
|
|
|
|
const isVideo = asset.mediaType?.startsWith('video/')
|
|
const isImage = asset.mediaType?.startsWith('image/')
|
|
if (!isVideo && !isImage) {
|
|
return
|
|
}
|
|
|
|
const teamId = params.teamId || asset.project?.teamId || null
|
|
const projectId = params.projectId || asset.projectId
|
|
|
|
if (!teamId) {
|
|
return
|
|
}
|
|
|
|
// Check if there is an active embedding agent for the team
|
|
const embeddingAgent = await prisma.agent.findFirst({
|
|
where: {
|
|
type: 'embedding',
|
|
enabled: true,
|
|
user: { teamMembers: { some: { teamId } } },
|
|
},
|
|
})
|
|
|
|
if (!embeddingAgent) {
|
|
return
|
|
}
|
|
|
|
// Check if a pending/processing embedding task already exists for this asset to avoid duplicates
|
|
const existing = await prisma.workflowTask.findFirst({
|
|
where: {
|
|
assetId: params.assetId,
|
|
type: WorkflowTaskType.ai_embedding,
|
|
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
|
},
|
|
})
|
|
if (existing) {
|
|
return
|
|
}
|
|
|
|
// Create embedding task
|
|
await prisma.workflowTask.create({
|
|
data: {
|
|
assetId: params.assetId,
|
|
type: WorkflowTaskType.ai_embedding,
|
|
status: WorkflowTaskStatus.pending,
|
|
teamId,
|
|
projectId,
|
|
payload: {
|
|
projectId: projectId ?? '',
|
|
agent: { agentId: embeddingAgent.id },
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
export interface CreateAutofillTaskIfEnabledParams {
|
|
assetId: string
|
|
teamId: string | null
|
|
projectId: string | null
|
|
}
|
|
|
|
export async function createAutofillTaskIfEnabledActivity(
|
|
params: CreateAutofillTaskIfEnabledParams,
|
|
): Promise<void> {
|
|
const asset = await prisma.asset.findUnique({
|
|
where: { id: params.assetId },
|
|
include: {
|
|
project: true,
|
|
},
|
|
})
|
|
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack')) {
|
|
return
|
|
}
|
|
|
|
const teamId = params.teamId || asset.project?.teamId || null
|
|
const projectId = params.projectId || asset.projectId
|
|
|
|
if (!teamId) {
|
|
return
|
|
}
|
|
|
|
// Check if there is an active autofill agent for the team
|
|
const autofillAgent = await prisma.agent.findFirst({
|
|
where: {
|
|
type: 'autofill',
|
|
enabled: true,
|
|
user: { teamMembers: { some: { teamId } } },
|
|
},
|
|
})
|
|
|
|
if (!autofillAgent) {
|
|
return
|
|
}
|
|
|
|
// Check if a pending/processing autofill task already exists for this asset to avoid duplicates
|
|
const existing = await prisma.workflowTask.findFirst({
|
|
where: {
|
|
assetId: params.assetId,
|
|
type: WorkflowTaskType.ai_metadata_autofill,
|
|
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
|
},
|
|
})
|
|
if (existing) {
|
|
return
|
|
}
|
|
|
|
// Create autofill task
|
|
await prisma.workflowTask.create({
|
|
data: {
|
|
assetId: params.assetId,
|
|
type: WorkflowTaskType.ai_metadata_autofill,
|
|
status: WorkflowTaskStatus.pending,
|
|
teamId,
|
|
projectId,
|
|
payload: {
|
|
projectId: projectId ?? '',
|
|
agent: { agentId: autofillAgent.id },
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
export interface TranscodeVideoChunkParams {
|
|
assetId: string
|
|
assetKey?: string
|
|
filePath?: string
|
|
startTime: number
|
|
endTime: number
|
|
}
|
|
|
|
export async function transcodeVideoChunkActivity(
|
|
params: TranscodeVideoChunkParams,
|
|
): Promise<{ chunkKey: string }> {
|
|
if (!params.filePath && !params.assetKey) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
|
|
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
|
|
try {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
const inputSource =
|
|
params.filePath ||
|
|
(params.assetKey ? await s3Service.resolveInput(bucket, params.assetKey) : '')
|
|
if (!inputSource) {
|
|
throw ApplicationFailure.create({
|
|
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
|
|
nonRetryable: true,
|
|
})
|
|
}
|
|
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
|
|
|
|
// Slice video segment using ffmpeg, force 1fps and remove audio for gemini-embedding-2 optimization
|
|
// Note: -ss placed before -i for fast input seeking via demuxer Range requests
|
|
await execFileAsync('ffmpeg', [
|
|
'-y',
|
|
'-loglevel',
|
|
'warning',
|
|
...(isRemote
|
|
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
|
|
: []),
|
|
'-ss',
|
|
params.startTime.toString(),
|
|
'-i',
|
|
inputSource,
|
|
'-map',
|
|
'0:V:0',
|
|
'-t',
|
|
(params.endTime - params.startTime).toString(),
|
|
'-vf',
|
|
'fps=1',
|
|
'-c:v',
|
|
'libx264',
|
|
'-preset',
|
|
'ultrafast',
|
|
'-crf',
|
|
'28',
|
|
'-an',
|
|
chunkTmp,
|
|
])
|
|
|
|
const chunkData = fs.readFileSync(chunkTmp)
|
|
|
|
// Upload to S3
|
|
const key = `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}-${ulid()}.mp4`
|
|
|
|
await s3Service.putObject(bucket, key, chunkData, chunkData.length, 'video/mp4')
|
|
|
|
return { chunkKey: key }
|
|
} catch (err) {
|
|
if (err instanceof ApplicationFailure) {
|
|
throw err
|
|
}
|
|
throw ApplicationFailure.create({
|
|
message: `Failed to transcode video chunk for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
|
|
nonRetryable: false,
|
|
})
|
|
} finally {
|
|
if (fs.existsSync(chunkTmp)) {
|
|
try {
|
|
fs.unlinkSync(chunkTmp)
|
|
} catch (e) {
|
|
console.error('Failed to cleanup local chunk file:', e)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function deleteS3ObjectActivity(params: { key: string }): Promise<void> {
|
|
const bucket = process.env.S3_BUCKET || 'shumai'
|
|
await s3Service.deleteObject(bucket, params.key)
|
|
}
|