Files
shumai/packages/transcode/src/activities/transcode.ts
T
Yiling-J fe8d404a44 feat(agent): support 3D degree comments, markup annotations, and config3d in read_asset (#473)
* feat(agent): support AI embedding, autofill, and read_asset for 3D assets

* feat(agent): support 3D degree comments, markup annotations, and config3d in read_asset

* fix(agent): resolve 3D session annotations to turntable degree frame time
2026-10-13 00:03:21 +08:00

1940 lines
57 KiB
TypeScript

import { AssetStatus, prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import {
transcodeService,
buildHlsMasterPlaylist,
calculateEffectiveBitrateBps,
type HdrType,
} from '@shumai/core/src/transcode/transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { getDerivedArtifactDirectory, stemFromKey } from '@shumai/core/src/utils/filename'
import {
gotenbergService,
type GotenbergConvertOptions,
} from '@shumai/core/src/gotenberg/gotenberg'
import { turntableService } from '@shumai/core/src/turntable/turntable'
import { parseCsvContent } from '@shumai/core/src/transcode/transcode'
import { resolutionToDimensions } from '../workflows/transcode-utils'
import {
getProxyType,
isCsvDocument,
isHtmlDocument,
isMarkdownDocument,
isOfficeDocument,
} from '@shumai/core/src/utils/mime'
import { logger } from '@shumai/core/src/logger'
import { ApplicationFailure, Context } from '@temporalio/activity'
import { getLocalTaskAbortSignal } from '@shumai/workflow-core'
function getActivityCancellationSignal(taskId?: string): AbortSignal | undefined {
try {
return Context.current().cancellationSignal
} catch {
return getLocalTaskAbortSignal(taskId)
}
}
import * as path from 'path'
import * as fs from 'fs'
import * as os from 'os'
import { execFile } from 'child_process'
import { promisify } from 'util'
import { ulid } from 'ulid'
const execFileAsync = promisify(execFile)
export interface GetMediaInfoActivityParams {
assetId: string
assetKey: string
mediaType: string
}
function getErrorDetails(err: unknown): { code?: string; message: string; name?: string } {
if (err && typeof err === 'object') {
const record = err as Record<string, unknown>
return {
code: typeof record.code === 'string' ? record.code : undefined,
message:
typeof record.message === 'string'
? record.message
: err instanceof Error
? err.message
: String(err),
name: typeof record.name === 'string' ? record.name : undefined,
}
}
return {
message: String(err),
}
}
async function ensureAssetNotPurging(assetKey: string): Promise<void> {
const sk = await prisma.storageKey.findUnique({
where: { key: assetKey },
include: { assets: { select: { id: true, status: true } } },
})
if (
sk &&
(sk.status === 'purging' ||
sk.assets.length === 0 ||
sk.assets.every((a) => a.status === AssetStatus.pending_purge))
) {
throw ApplicationFailure.create({
message: 'Asset or storage key has been purged or is pending purge',
nonRetryable: true,
})
}
}
export async function getMediaInfoActivity(params: {
filePath: string
assetId: string
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
mediaType?: string
}): Promise<PrismaJson.MediaInfo> {
try {
const proxyType =
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
const fileType = isVideo
? 'video'
: isAudio
? 'audio'
: isImage
? 'image'
: isPdf
? 'document'
: 'file'
const mediaInfo: PrismaJson.MediaInfo = {
duration: 0,
filesize: 0,
frames: 0,
proxyType,
imageTranscodes: [],
videoTranscodes: [],
videoPreview: { width: 0, height: 0 },
finishedAt: new Date().toISOString(),
metadata: null,
original: {
key: '', // Will be filled by caller or updated later
filesizeInBytes: 0,
codec: '',
},
}
const metadataUpdates: { key: string; value: string | number }[] = [
{ key: 'file_type', value: fileType },
]
if (proxyType) {
metadataUpdates.push({ key: 'proxy_type', value: proxyType })
}
if (isVideo) {
let isDowngradedToAudio = false
let videoInfo: Awaited<ReturnType<typeof transcodeService.getVideoInfo>> | null = null
try {
videoInfo = await transcodeService.getVideoInfo(params.filePath)
} catch (err) {
if (err instanceof Error && err.message.includes('No video stream found')) {
try {
const audioInfo = await transcodeService.getAudioInfo(params.filePath)
isDowngradedToAudio = true
mediaInfo.proxyType = 'audio'
mediaInfo.duration = audioInfo.duration
mediaInfo.metadata = {
originalWidth: 0,
originalHeight: 0,
duration: audioInfo.duration,
bitRate: audioInfo.bitRate,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: true,
videoCodec: undefined,
audioCodec: audioInfo.audioCodec,
audioChannels: audioInfo.audioChannels,
audioSampleRate: audioInfo.audioSampleRate,
audioBitDepth: audioInfo.audioBitDepth,
audioStreamIndex: audioInfo.audioStreamIndex,
format: {},
}
const ftIdx = metadataUpdates.findIndex((u) => u.key === 'file_type')
if (ftIdx !== -1) metadataUpdates[ftIdx].value = 'audio'
const ptIdx = metadataUpdates.findIndex((u) => u.key === 'proxy_type')
if (ptIdx !== -1) metadataUpdates[ptIdx].value = 'audio'
metadataUpdates.push(
{ key: 'duration', value: audioInfo.duration },
{ key: 'bitRate', value: audioInfo.bitRate / 1000 },
)
if (audioInfo.audioCodec)
metadataUpdates.push({ key: 'audio_codec', value: audioInfo.audioCodec })
if (audioInfo.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: audioInfo.audioChannels })
if (audioInfo.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: audioInfo.audioSampleRate })
if (audioInfo.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: audioInfo.audioBitDepth })
} catch {
throw err
}
} else {
throw err
}
}
if (!isDowngradedToAudio && videoInfo) {
const info = videoInfo
mediaInfo.duration = info.duration
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: info.duration,
bitRate: info.bitRate,
videoBitRate: info.videoBitRate,
frameRate: info.frameRate,
totalFrames: info.totalFrames,
startTimecode: info.startTimecode || '00:00:00:00',
hasAudio: info.hasAudio,
videoCodec: info.videoCodec,
audioCodec: info.audioCodec,
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
colorTransfer: info.colorTransfer,
colorPrimaries: info.colorPrimaries,
colorSpace: info.colorSpace,
isHdr: info.isHdr,
hdrType: info.hdrType,
dvProfile: info.dvProfile,
rotation: info.rotation,
videoStreamIndex: info.videoStreamIndex,
audioStreamIndex: info.audioStreamIndex,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'duration', value: info.duration },
{ key: 'bitRate', value: info.bitRate / 1000 },
{ key: 'frame_rate', value: info.frameRate },
)
if (info.rotation !== undefined)
metadataUpdates.push({ key: 'rotation', value: info.rotation })
if (info.isHdr !== undefined)
metadataUpdates.push({ key: 'is_hdr', value: info.isHdr ? 1 : 0 })
if (info.colorTransfer)
metadataUpdates.push({ key: 'color_transfer', value: info.colorTransfer })
if (info.videoCodec) metadataUpdates.push({ key: 'video_codec', value: info.videoCodec })
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
if (info.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
if (info.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
if (info.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
}
} else if (isAudio) {
const info = await transcodeService.getAudioInfo(params.filePath)
mediaInfo.duration = info.duration
mediaInfo.metadata = {
originalWidth: 0,
originalHeight: 0,
duration: info.duration,
bitRate: info.bitRate,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: true,
videoCodec: undefined,
audioCodec: info.audioCodec,
audioChannels: info.audioChannels,
audioSampleRate: info.audioSampleRate,
audioBitDepth: info.audioBitDepth,
audioStreamIndex: info.audioStreamIndex,
format: {},
}
metadataUpdates.push(
{ key: 'duration', value: info.duration },
{ key: 'bitRate', value: info.bitRate / 1000 },
)
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
if (info.audioChannels !== undefined)
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
if (info.audioSampleRate !== undefined)
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
if (info.audioBitDepth !== undefined)
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
} else if (isImage) {
const info = await transcodeService.getImageInfo(params.filePath)
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: 0,
startTimecode: '00:00:00:00',
hasAudio: false,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
)
} else if (isPdf) {
const info = await transcodeService.getPdfInfo(params.filePath)
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: info.totalFrames,
startTimecode: '00:00:00:00',
hasAudio: false,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'total_pages', value: info.totalFrames },
)
}
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
return mediaInfo
} catch (err) {
const { name, message } = getErrorDetails(err)
if (name?.includes('Prisma') || message.includes('Prisma')) {
throw err
}
throw ApplicationFailure.create({
message: `Failed to get media info: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
}
export interface VideoActivityParams {
taskId?: string
assetKey: string
filePath: string
videoSpec: PrismaJson.VideoTranscode
duration: number
originalFps: number
hardwareAcceleration?: PrismaJson.HardwareAcceleration
sourceVideoBitrate?: number
threads?: number
sourceIsHdr?: boolean
sourceHdrType?: HdrType
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
sourceRotation?: number
streamIndex?: number
audioStreamIndex?: number
}
export async function transcodeVideoActivity(
params: VideoActivityParams,
): Promise<PrismaJson.VideoTranscode> {
const bucket = process.env.S3_BUCKET || 'shumai'
const stem = stemFromKey(params.assetKey)
const res = params.videoSpec.resolution || `${params.videoSpec.height}p`
const suffix = params.videoSpec.hdr ? `${res}-hdr` : res
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-${suffix}.mp4`)
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Video transcoding cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, key)
return { ...params.videoSpec, key }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const outputFile = path.join(tmpDir, `${stem}-${suffix}.mp4`)
try {
let targetFps: number | string = params.originalFps || 30
let disableAudio = false
if (params.videoSpec.resolution === '180p') {
disableAudio = true
targetFps = 24.0
if (params.originalFps > 0 && params.originalFps < 24.0) {
targetFps = params.originalFps
}
if (params.duration * (targetFps as number) > 1600) {
targetFps = `160000/${Math.floor(params.duration * 100)}`
}
}
await transcodeService.transcodeVideo({
inputFile: params.filePath,
outputFile,
width: params.videoSpec.width,
height: params.videoSpec.height,
frameRate: targetFps || undefined,
disableAudio,
hardwareAcceleration: params.hardwareAcceleration,
sourceVideoBitrate: params.sourceVideoBitrate,
threads: params.threads,
signal,
hdr: params.videoSpec.hdr,
sourceIsHdr: params.sourceIsHdr,
sourceHdrType: params.sourceHdrType,
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
sourceRotation: params.sourceRotation,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
const stat = fs.statSync(outputFile)
const stream = Bun.file(outputFile).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
return { ...params.videoSpec, key }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Video transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('no video stream found') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `Video transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
}
}
export interface HlsActivityParams {
taskId: string
assetKey: string
filePath: string
ladders: PrismaJson.HlsResolutionLadder[]
originalWidth: number
originalHeight: number
duration: number
originalFps: number
hardwareAcceleration?: 'off' | 'auto'
sourceVideoBitrate?: number
threads?: number
sourceIsHdr?: boolean
sourceHdrType?: HdrType
sourceColorTransfer?: string
sourceColorPrimaries?: string
sourceColorSpace?: string
sourceRotation?: number
streamIndex?: number
audioStreamIndex?: number
}
export async function transcodeHlsActivity(params: HlsActivityParams): Promise<PrismaJson.HlsInfo> {
const bucket = process.env.S3_BUCKET || 'shumai'
const assetDir = path.posix.dirname(params.assetKey)
const hlsDirKey = path.posix.join(assetDir, 'hls')
const masterKey = path.posix.join(hlsDirKey, 'master.m3u8')
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'HLS transcoding cancelled',
nonRetryable: true,
})
}
const hlsTmpDir = path.join(path.dirname(params.filePath), `hls-${ulid()}`)
try {
const renditionSpecs: Array<{
resolution: string
width: number
height: number
bitrateBps: number
isHdr?: boolean
}> = []
const mediaResolutions: PrismaJson.VideoTranscode[] = []
for (const ladder of params.ladders) {
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'HLS transcoding cancelled',
nonRetryable: true,
})
}
const [width, height] = resolutionToDimensions(
ladder,
params.originalWidth,
params.originalHeight,
)
const targetBps = calculateEffectiveBitrateBps(
height,
width,
params.sourceVideoBitrate,
params.originalFps,
)
const ladderDir = path.join(hlsTmpDir, ladder)
await transcodeService.transcodeHlsRendition({
inputFile: params.filePath,
outputDir: ladderDir,
width,
height,
frameRate: params.originalFps,
hardwareAcceleration: params.hardwareAcceleration,
sourceVideoBitrate: params.sourceVideoBitrate,
threads: params.threads,
signal,
hdr: Boolean(params.sourceIsHdr),
sourceIsHdr: params.sourceIsHdr,
sourceHdrType: params.sourceHdrType,
sourceColorTransfer: params.sourceColorTransfer,
sourceColorPrimaries: params.sourceColorPrimaries,
sourceColorSpace: params.sourceColorSpace,
sourceRotation: params.sourceRotation,
streamIndex: params.streamIndex,
audioStreamIndex: params.audioStreamIndex,
})
renditionSpecs.push({
resolution: ladder,
width,
height,
bitrateBps: targetBps,
isHdr: params.sourceIsHdr,
})
mediaResolutions.push({
resolution: ladder,
width,
height,
hdr: Boolean(params.sourceIsHdr),
})
await ensureAssetNotPurging(params.assetKey)
const files = fs.readdirSync(ladderDir)
for (const file of files) {
const localFilePath = path.join(ladderDir, file)
const stat = fs.statSync(localFilePath)
const s3Key = path.posix.join(hlsDirKey, ladder, file)
let contentType = 'application/octet-stream'
if (file.endsWith('.m3u8')) {
contentType = 'application/vnd.apple.mpegurl'
} else if (file.endsWith('.mp4')) {
contentType = 'video/mp4'
} else if (file.endsWith('.m4s')) {
contentType = 'video/iso.segment'
}
const stream = Bun.file(localFilePath).stream()
await s3Service.putObject(bucket, s3Key, stream, stat.size, contentType)
}
}
const masterContent = buildHlsMasterPlaylist(renditionSpecs)
const masterBuffer = Buffer.from(masterContent, 'utf-8')
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
masterKey,
masterBuffer,
masterBuffer.length,
'application/vnd.apple.mpegurl',
)
return {
key: masterKey,
resolutions: mediaResolutions,
}
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'HLS transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('no video stream found') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `HLS transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(hlsTmpDir)) {
try {
fs.rmSync(hlsTmpDir, { recursive: true, force: true })
} catch {
// ignore cleanup error
}
}
}
}
export interface AudioActivityParams {
taskId?: string
assetKey: string
filePath: string
threads?: number
audioStreamIndex?: number
}
export async function transcodeAudioActivity(
params: AudioActivityParams,
): Promise<PrismaJson.VideoTranscode> {
const bucket = process.env.S3_BUCKET || 'shumai'
const stem = stemFromKey(params.assetKey)
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Audio transcoding cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, key)
return { width: 0, height: 0, key }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const outputFile = path.join(tmpDir, `${stem}-audio-proxy.mp4`)
try {
await transcodeService.transcodeAudio({
inputFile: params.filePath,
outputFile,
bitrate: '128k',
threads: params.threads,
signal,
audioStreamIndex: params.audioStreamIndex,
})
const stat = fs.statSync(outputFile)
const stream = Bun.file(outputFile).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
return { width: 0, height: 0, key }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Audio transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('no video stream found')
) {
throw ApplicationFailure.create({
message: `Audio transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
}
}
export interface ImageActivityParams {
assetKey: string
filePath: string
imageSpec: PrismaJson.ImageTranscode
}
export async function transcodeImageActivity(
params: ImageActivityParams,
): Promise<PrismaJson.ImageTranscode> {
const bucket = process.env.S3_BUCKET || 'shumai'
const stem = stemFromKey(params.assetKey)
const key = path.posix.join(
path.posix.dirname(params.assetKey),
`${stem}-${params.imageSpec.width}p.webp`,
)
try {
await s3Service.headObject(bucket, key)
return { ...params.imageSpec, key }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const outputFile = path.join(tmpDir, `${stem}-${params.imageSpec.width}p.webp`)
try {
await transcodeService.transcodeImage(
params.filePath,
outputFile,
params.imageSpec.width,
params.imageSpec.quality,
{
height: params.imageSpec.height || null,
isPreview: params.imageSpec.isPreview ?? false,
},
)
const buffer = fs.readFileSync(outputFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
return { ...params.imageSpec, key }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('sharp') ||
lowerMsg.includes('imagemagick') ||
lowerMsg.includes('magick') ||
lowerMsg.includes('convert') ||
lowerMsg.includes('dcraw') ||
lowerMsg.includes('raw') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format')
) {
throw ApplicationFailure.create({
message: `Image transcoding failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
}
}
export interface GenerateSpriteActivityParams {
taskId?: string
assetKey: string
filePath: string
mediaInfo: PrismaJson.MediaInfo
spriteSpec: PrismaJson.SpriteInfo
posterSpec: PrismaJson.PosterInfo
}
export async function generateSpriteActivity(params: GenerateSpriteActivityParams) {
const bucket = process.env.S3_BUCKET || 'shumai'
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Sprite/Poster generation cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, params.spriteSpec.key)
await s3Service.headObject(bucket, params.posterSpec.key)
return { sprite: params.spriteSpec, poster: params.posterSpec }
} catch {
// Not found
}
const tmpDir = path.dirname(params.filePath)
const spriteFile = path.join(tmpDir, 'sprite.webp')
const posterFile = path.join(tmpDir, 'poster.webp')
try {
if (params.mediaInfo.proxyType === 'pdf') {
const pdfRes = await transcodeService.generatePdfSprite(
params.filePath,
spriteFile,
posterFile,
signal,
)
if (params.mediaInfo && params.mediaInfo.metadata) {
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
params.mediaInfo.frames = pdfRes.pageCount
}
} else {
await transcodeService.generateSprite(
params.filePath,
spriteFile,
posterFile,
params.mediaInfo.duration,
signal,
{
isHdr: params.mediaInfo.metadata?.isHdr,
hdrType: params.mediaInfo.metadata?.hdrType,
colorTransfer: params.mediaInfo.metadata?.colorTransfer,
},
params.mediaInfo.metadata?.videoStreamIndex,
)
}
const spriteBuffer = fs.readFileSync(spriteFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.spriteSpec.key,
spriteBuffer,
spriteBuffer.length,
'image/webp',
)
const posterBuffer = fs.readFileSync(posterFile)
await s3Service.putObject(
bucket,
params.posterSpec.key,
posterBuffer,
posterBuffer.length,
'image/webp',
)
return { sprite: params.spriteSpec, poster: params.posterSpec }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Sprite/Poster generation cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('ffprobe') ||
lowerMsg.includes('spawn') ||
lowerMsg.includes('format') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `Sprite/Poster generation failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
if (fs.existsSync(spriteFile)) fs.unlinkSync(spriteFile)
if (fs.existsSync(posterFile)) fs.unlinkSync(posterFile)
}
}
export interface GeneratePdfProxyActivityParams {
teamId?: string | null
assetId: string
assetKey: string
filePath: string
mediaType: string
filename: string
}
export async function generatePdfProxyActivity(
params: GeneratePdfProxyActivityParams,
): Promise<{ pdfProxyKey: string; pdfFilePath: string }> {
const isOffice = isOfficeDocument(params.mediaType, params.filename)
const isHtml = isHtmlDocument(params.mediaType, params.filename)
const isMd = isMarkdownDocument(params.mediaType, params.filename)
const isCsv = isCsvDocument(params.mediaType, params.filename)
const isTxt = params.mediaType === 'text/plain' || params.filename.toLowerCase().endsWith('.txt')
if (!isOffice && !isHtml && !isMd && !isCsv && !isTxt) {
return { pdfProxyKey: params.assetKey, pdfFilePath: params.filePath }
}
const bucket = process.env.S3_BUCKET || 'shumai'
const assetDir = getDerivedArtifactDirectory(params.assetKey, params.assetId)
const pdfProxyKey = `${assetDir}/proxy.pdf`
const tmpDir = path.dirname(params.filePath)
const pdfFilePath = path.join(tmpDir, 'proxy.pdf')
const targetTeamId = params.teamId || undefined
try {
const gotenbergAvailable = await (targetTeamId
? gotenbergService.isAvailable(targetTeamId)
: gotenbergService.isAvailable())
if (isOffice) {
if (!gotenbergAvailable) {
throw new Error(
'Gotenberg service is required for Office document conversion but is unavailable.',
)
}
const pdfBuffer = await (targetTeamId
? gotenbergService.convertDocumentToPdf(
params.filePath,
params.filename,
{ teamId: targetTeamId },
targetTeamId,
)
: gotenbergService.convertDocumentToPdf(params.filePath, params.filename))
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else if (isHtml) {
if (!gotenbergAvailable) {
throw new Error(
'Gotenberg service is required for HTML document conversion but is unavailable.',
)
}
const pdfBuffer = await (targetTeamId
? gotenbergService.convertHtmlToPdf(params.filePath, params.filename, targetTeamId)
: gotenbergService.convertHtmlToPdf(params.filePath, params.filename))
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else if (isCsv) {
if (gotenbergAvailable) {
let isLandscape = false
try {
const content = String(fs.readFileSync(params.filePath, 'utf-8'))
const rows = parseCsvContent(content)
const maxCols = Math.max(...rows.map((r) => r.length), 0)
isLandscape = maxCols > 5
} catch {
// Fallback landscape calculation
}
const options: GotenbergConvertOptions = {
landscape: isLandscape,
...(targetTeamId ? { teamId: targetTeamId } : {}),
}
const pdfBuffer = await gotenbergService.convertDocumentToPdf(
params.filePath,
params.filename,
options,
)
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else {
await transcodeService.generatePdfFromCsv(params.filePath, pdfFilePath)
}
} else if (isMd) {
if (gotenbergAvailable) {
const pdfBuffer = await (targetTeamId
? gotenbergService.convertMarkdownToPdf(params.filePath, params.filename, targetTeamId)
: gotenbergService.convertMarkdownToPdf(params.filePath, params.filename))
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else {
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
}
} else if (isTxt) {
if (gotenbergAvailable) {
const pdfBuffer = await (targetTeamId
? gotenbergService.convertDocumentToPdf(
params.filePath,
params.filename,
{ teamId: targetTeamId },
targetTeamId,
)
: gotenbergService.convertDocumentToPdf(params.filePath, params.filename))
fs.writeFileSync(pdfFilePath, pdfBuffer)
} else {
await transcodeService.generatePdfFromText(params.filePath, pdfFilePath)
}
}
const stat = fs.statSync(pdfFilePath)
const stream = Bun.file(pdfFilePath).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, pdfProxyKey, stream, stat.size, 'application/pdf')
return { pdfProxyKey, pdfFilePath }
} catch (err) {
const { message } = getErrorDetails(err)
throw ApplicationFailure.create({
message: `Failed to generate PDF proxy: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
}
export async function downloadMediaToTmpActivity(params: {
assetKey: string
}): Promise<{ filePath: string; tmpDir: string }> {
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = transcodeService.createTempDir('transcode-')
const filePath = path.join(tmpDir, path.basename(params.assetKey))
if (!fs.existsSync(tmpDir)) {
fs.mkdirSync(tmpDir, { recursive: true })
}
try {
await s3Service.downloadToFile(bucket, params.assetKey, filePath)
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (code === 'ENOENT' || lowerMsg.includes('enoent') || lowerMsg.includes('nosuchkey')) {
throw ApplicationFailure.create({
message: `Failed to download media to tmp: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
return { filePath, tmpDir }
}
export async function cleanupTmpDirActivity(params: { tmpDir: string }): Promise<void> {
transcodeService.removeDir(params.tmpDir)
}
export interface UpdateAssetMediaActivityParams {
assetId: string
mediaInfo: PrismaJson.MediaInfo
}
export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityParams) {
const bucket = process.env.S3_BUCKET || 'shumai'
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: { storageKey: true },
})
const key = asset?.storageKey?.key
if (!asset || !key || asset.status === AssetStatus.pending_purge) {
throw ApplicationFailure.create({
message: 'Asset not found, has no key, or is being purged',
nonRetryable: true,
})
}
const infoKey = path.join(path.dirname(key), 'info.json')
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
const { count } = await prisma.asset.updateMany({
where: {
id: params.assetId,
status: { not: AssetStatus.pending_purge },
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: { media: params.mediaInfo as any, hasJpegPreview: false },
})
if (count === 0) {
logger.info(
{ assetId: params.assetId },
'[updateAssetMediaActivity] Asset was deleted or pending purge before media update completed',
)
}
}
export async function takeScreenshotsActivity(params: {
taskId?: string
assetKey: string
assetId: string
start: number
end: number
count: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; timestamp: number }>> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Screenshot extraction cancelled',
nonRetryable: true,
})
}
try {
return await transcodeService.takeScreenshots({ ...params, signal })
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Screenshot extraction cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `Screenshot extraction failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export interface ExtractPosterActivityParams {
assetKey: string
posterSpec: PrismaJson.PosterInfo
taskId?: string
signal?: AbortSignal
isHdr?: boolean
hdrType?: HdrType
colorTransfer?: string
streamIndex?: number
}
export async function extractPosterActivity(
params: ExtractPosterActivityParams,
): Promise<{ poster: PrismaJson.PosterInfo }> {
const signal = params.signal || getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Poster extraction cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
try {
await s3Service.headObject(bucket, params.posterSpec.key)
return { poster: params.posterSpec }
} catch {
// Not found, proceed to extract
}
const tmpDir = transcodeService.createTempDir('poster-')
const posterFile = path.join(tmpDir, 'poster.webp')
try {
const inputSource = await s3Service.resolveInput(bucket, params.assetKey)
await transcodeService.generatePoster(inputSource, posterFile, {
isHdr: params.isHdr,
hdrType: params.hdrType,
colorTransfer: params.colorTransfer,
signal,
streamIndex: params.streamIndex,
})
const posterBuffer = fs.readFileSync(posterFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.posterSpec.key,
posterBuffer,
posterBuffer.length,
'image/webp',
)
return { poster: params.posterSpec }
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Poster extraction cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('sharp') ||
lowerMsg.includes('zscale') ||
lowerMsg.includes('tonemap')
) {
throw ApplicationFailure.create({
message: `Poster extraction failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
} finally {
transcodeService.removeDir(tmpDir)
}
}
export async function overlayAnnotationsActivity(params: {
assetKey: string
assetId: string
annotations: PrismaJson.AnnotationList
}): Promise<string> {
try {
return await transcodeService.overlayAnnotations(params)
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('ffmpeg') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `Overlay annotations failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export async function renderPdfPagesActivity(params: {
taskId?: string
assetKey: string
assetId: string
start: number
end: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; page: number }>> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'PDF page rendering cancelled',
nonRetryable: true,
})
}
try {
return await transcodeService.renderPdfPages({ ...params, signal })
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'PDF page rendering cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
lowerMsg.includes('nosuchkey') ||
lowerMsg.includes('pdftoppm') ||
lowerMsg.includes('sharp')
) {
throw ApplicationFailure.create({
message: `PDF page rendering failed: ${message}`,
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
throw err
}
}
export interface CreateEmbeddingTaskIfEnabledParams {
assetId: string
teamId: string | null
projectId: string | null
}
export async function createEmbeddingTaskIfEnabledActivity(
params: CreateEmbeddingTaskIfEnabledParams,
): Promise<void> {
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: {
project: true,
},
})
if (!asset) {
return
}
const mediaInfo = asset.media as PrismaJson.MediaInfo | null
const isVideo = asset.mediaType?.startsWith('video/')
const isImage =
asset.mediaType?.startsWith('image/') ||
(mediaInfo?.proxyType === '3d' && (mediaInfo.imageTranscodes?.length ?? 0) > 0)
if (!isVideo && !isImage) {
return
}
const teamId = params.teamId || asset.project?.teamId || null
const projectId = params.projectId || asset.projectId
if (!teamId) {
return
}
// Check if there is an active embedding agent for the team
const embeddingAgent = await prisma.agent.findFirst({
where: {
type: 'embedding',
enabled: true,
user: { teamMembers: { some: { teamId } } },
},
})
if (!embeddingAgent) {
return
}
// Check if a pending/processing embedding task already exists for this asset to avoid duplicates
const existing = await prisma.workflowTask.findFirst({
where: {
assetId: params.assetId,
type: WorkflowTaskType.ai_embedding,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
})
if (existing) {
return
}
// Create embedding task
await prisma.workflowTask.create({
data: {
assetId: params.assetId,
type: WorkflowTaskType.ai_embedding,
status: WorkflowTaskStatus.pending,
teamId,
projectId,
payload: {
projectId: projectId ?? '',
agent: { agentId: embeddingAgent.id },
},
},
})
}
export interface CreateAutofillTaskIfEnabledParams {
assetId: string
teamId: string | null
projectId: string | null
}
export async function createAutofillTaskIfEnabledActivity(
params: CreateAutofillTaskIfEnabledParams,
): Promise<void> {
const asset = await prisma.asset.findUnique({
where: { id: params.assetId },
include: {
project: true,
},
})
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack')) {
return
}
const teamId = params.teamId || asset.project?.teamId || null
const projectId = params.projectId || asset.projectId
if (!teamId) {
return
}
// Check if there is an active autofill agent for the team
const autofillAgent = await prisma.agent.findFirst({
where: {
type: 'autofill',
enabled: true,
user: { teamMembers: { some: { teamId } } },
},
})
if (!autofillAgent) {
return
}
// Check if a pending/processing autofill task already exists for this asset to avoid duplicates
const existing = await prisma.workflowTask.findFirst({
where: {
assetId: params.assetId,
type: WorkflowTaskType.ai_metadata_autofill,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
})
if (existing) {
return
}
// Create autofill task
await prisma.workflowTask.create({
data: {
assetId: params.assetId,
type: WorkflowTaskType.ai_metadata_autofill,
status: WorkflowTaskStatus.pending,
teamId,
projectId,
payload: {
projectId: projectId ?? '',
agent: { agentId: autofillAgent.id },
},
},
})
}
export interface TranscodeVideoChunkParams {
assetId: string
assetKey?: string
filePath?: string
startTime: number
endTime: number
}
export async function transcodeVideoChunkActivity(
params: TranscodeVideoChunkParams,
): Promise<{ chunkKey: string }> {
if (!params.filePath && !params.assetKey) {
throw ApplicationFailure.create({
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
nonRetryable: true,
})
}
const chunkTmp = path.join(os.tmpdir(), `video-chunk-${Date.now()}-${ulid()}.mp4`)
try {
const bucket = process.env.S3_BUCKET || 'shumai'
const inputSource =
params.filePath ||
(params.assetKey ? await s3Service.resolveInput(bucket, params.assetKey) : '')
if (!inputSource) {
throw ApplicationFailure.create({
message: 'Neither filePath nor assetKey was provided for video chunk transcoding',
nonRetryable: true,
})
}
const isRemote = inputSource.startsWith('http://') || inputSource.startsWith('https://')
// Slice video segment using ffmpeg, force 1fps and remove audio for gemini-embedding-2 optimization
// Note: -ss placed before -i for fast input seeking via demuxer Range requests
await execFileAsync('ffmpeg', [
'-y',
'-loglevel',
'warning',
...(isRemote
? ['-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5']
: []),
'-ss',
params.startTime.toString(),
'-i',
inputSource,
'-map',
'0:V:0',
'-t',
(params.endTime - params.startTime).toString(),
'-vf',
'fps=1',
'-c:v',
'libx264',
'-preset',
'ultrafast',
'-crf',
'28',
'-an',
chunkTmp,
])
const chunkData = fs.readFileSync(chunkTmp)
// Upload to S3
const key = `files/${params.assetId}/tmp-embedding-chunks/chunk-${params.startTime}-${params.endTime}-${ulid()}.mp4`
await s3Service.putObject(bucket, key, chunkData, chunkData.length, 'video/mp4')
return { chunkKey: key }
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
throw ApplicationFailure.create({
message: `Failed to transcode video chunk for ${params.startTime}-${params.endTime}: ${err instanceof Error ? err.message : String(err)}`,
nonRetryable: false,
})
} finally {
if (fs.existsSync(chunkTmp)) {
try {
fs.unlinkSync(chunkTmp)
} catch (e) {
console.error('Failed to cleanup local chunk file:', e)
}
}
}
}
export async function deleteS3ObjectActivity(params: { key: string }): Promise<void> {
const bucket = process.env.S3_BUCKET || 'shumai'
await s3Service.deleteObject(bucket, params.key)
}
export interface Render3dModelActivityParams {
taskId: string
teamId: string
assetId: string
assetKey: string
filePath: string
filename: string
targetVideoKey: string
targetPosterKey: string
}
export interface Render3dModelActivityResult {
videoFilePath: string
posterFilePath: string
modelMetadata?: Record<string, unknown>
}
export interface Start3dRenderActivityParams {
taskId: string
teamId: string
assetId: string
assetKey: string
filePath: string
filename: string
targetPosterKey: string
}
export interface Start3dRenderActivityResult {
renderTaskId: string
posterFilePath: string
posterWidth?: number
posterHeight?: number
modelMetadata?: Record<string, unknown>
}
export async function start3dRenderActivity(
params: Start3dRenderActivityParams,
): Promise<Start3dRenderActivityResult> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Render cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = path.dirname(params.filePath)
const posterFilePath = path.join(tmpDir, 'poster.webp')
let fileId: string | undefined
let renderTaskId: string | undefined
try {
const uploadResult = await turntableService.uploadFile(
params.filePath,
params.filename,
params.teamId,
)
fileId = uploadResult.id
if (signal?.aborted) {
await turntableService.deleteFile(fileId, params.teamId).catch(() => {})
throw ApplicationFailure.create({
message: 'Render cancelled',
nonRetryable: true,
})
}
const startResult = await turntableService.startRender(fileId, {}, params.teamId)
renderTaskId = startResult.taskId
const posterFileId = startResult.poster?.fileId
if (!posterFileId) {
throw ApplicationFailure.create({
message: 'Poster file ID missing from turntable render result',
nonRetryable: true,
})
}
const posterBuffer = await turntableService.downloadFile(posterFileId, undefined, params.teamId)
const posterWebpBuffer = await transcodeService.convertImageToWebp(posterBuffer)
fs.writeFileSync(posterFilePath, posterWebpBuffer)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.targetPosterKey,
posterWebpBuffer,
posterWebpBuffer.length,
'image/webp',
)
return {
renderTaskId,
posterFilePath,
posterWidth: startResult.poster?.width,
posterHeight: startResult.poster?.height,
modelMetadata: (startResult.metadata as Record<string, unknown>) || {},
}
} catch (err) {
if (renderTaskId) {
await turntableService.deleteTask(renderTaskId, params.teamId).catch(() => {})
} else if (fileId) {
await turntableService.deleteFile(fileId, params.teamId).catch(() => {})
}
if (err instanceof ApplicationFailure) {
throw err
}
const errMsg = err instanceof Error ? err.message : String(err)
throw ApplicationFailure.create({
message: `Failed to start 3D render: ${errMsg}`,
nonRetryable: true,
})
}
}
export interface WaitFor3dVideoRenderActivityParams {
taskId: string
teamId: string
assetKey: string
renderTaskId: string
targetVideoKey: string
tmpDir: string
}
export interface WaitFor3dVideoRenderActivityResult {
videoFilePath: string
}
export async function waitFor3dVideoRenderActivity(
params: WaitFor3dVideoRenderActivityParams,
): Promise<WaitFor3dVideoRenderActivityResult> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
await turntableService.deleteTask(params.renderTaskId, params.teamId).catch(() => {})
throw ApplicationFailure.create({
message: 'Render cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
const videoFilePath = path.join(params.tmpDir, 'turntable.mp4')
try {
let finishedStatus: 'completed' | 'failed' | null = null
let taskRecord: Awaited<ReturnType<typeof turntableService.getTaskStatus>> | null = null
for (let i = 0; i < 360; i++) {
if (signal?.aborted) {
await turntableService.deleteTask(params.renderTaskId, params.teamId).catch(() => {})
throw ApplicationFailure.create({
message: 'Render cancelled',
nonRetryable: true,
})
}
if (i > 0) {
await new Promise((resolve) => setTimeout(resolve, 5000))
}
taskRecord = await turntableService.getTaskStatus(params.renderTaskId, params.teamId)
if (taskRecord.status === 'completed') {
finishedStatus = 'completed'
break
}
if (taskRecord.status === 'failed') {
finishedStatus = 'failed'
break
}
}
if (finishedStatus !== 'completed' || !taskRecord) {
const errMsg =
finishedStatus === 'failed'
? taskRecord?.error?.message || '3D render failed on turntable server'
: '3D render timed out on turntable server'
throw ApplicationFailure.create({
message: errMsg,
nonRetryable: true,
})
}
const videoFileId = taskRecord.video?.fileId
if (!videoFileId) {
throw ApplicationFailure.create({
message: 'Video file ID missing from turntable render result',
nonRetryable: true,
})
}
const videoBuffer = await turntableService.downloadFile(
videoFileId,
videoFilePath,
params.teamId,
)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.targetVideoKey,
videoBuffer,
videoBuffer.length,
'video/mp4',
)
await turntableService.deleteTask(params.renderTaskId, params.teamId).catch((err) => {
logger.warn(
{ err, taskId: params.renderTaskId },
'Failed to delete turntable task after completion',
)
})
return {
videoFilePath,
}
} catch (err) {
await turntableService.deleteTask(params.renderTaskId, params.teamId).catch(() => {})
if (err instanceof ApplicationFailure) {
throw err
}
const errMsg = err instanceof Error ? err.message : String(err)
throw ApplicationFailure.create({
message: `Failed to render 3D model video: ${errMsg}`,
nonRetryable: true,
})
}
}
/**
* @deprecated Legacy monolithic 3D render activity. Prefer using `start3dRenderActivity` and `waitFor3dVideoRenderActivity`.
*/
export async function render3dModelActivity(
params: Render3dModelActivityParams,
): Promise<Render3dModelActivityResult> {
const tmpDir = path.dirname(params.filePath)
const startResult = await start3dRenderActivity({
taskId: params.taskId,
teamId: params.teamId,
assetId: params.assetId,
assetKey: params.assetKey,
filePath: params.filePath,
filename: params.filename,
targetPosterKey: params.targetPosterKey,
})
const videoResult = await waitFor3dVideoRenderActivity({
taskId: params.taskId,
teamId: params.teamId,
assetKey: params.assetKey,
renderTaskId: startResult.renderTaskId,
targetVideoKey: params.targetVideoKey,
tmpDir,
})
return {
videoFilePath: videoResult.videoFilePath,
posterFilePath: startResult.posterFilePath,
modelMetadata: startResult.modelMetadata,
}
}
export interface Generate3dSpriteActivityParams {
taskId: string
assetKey: string
videoFilePath: string
spriteKey: string
}
export async function generate3dSpriteActivity(
params: Generate3dSpriteActivityParams,
): Promise<{ spriteKey: string }> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Sprite generation cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = path.dirname(params.videoFilePath)
const spriteFilePath = path.join(tmpDir, 'sprite.webp')
try {
await transcodeService.generate3dSprite(params.videoFilePath, spriteFilePath, signal)
const spriteBuffer = fs.readFileSync(spriteFilePath)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.spriteKey,
spriteBuffer,
spriteBuffer.length,
'image/webp',
)
return { spriteKey: params.spriteKey }
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
const errMsg = err instanceof Error ? err.message : String(err)
throw ApplicationFailure.create({
message: `Failed to generate 3D sprite: ${errMsg}`,
nonRetryable: true,
})
}
}
export interface Generate3dViewsActivityParams {
taskId: string
assetKey: string
videoFilePath: string
viewsKey: string
}
export async function generate3dViewsActivity(
params: Generate3dViewsActivityParams,
): Promise<{ viewsKey: string }> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: '3D views generation cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = path.dirname(params.videoFilePath)
const viewsFilePath = path.join(tmpDir, 'views.webp')
try {
await transcodeService.generate3dViews(params.videoFilePath, viewsFilePath, signal)
const viewsBuffer = fs.readFileSync(viewsFilePath)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.viewsKey,
viewsBuffer,
viewsBuffer.length,
'image/webp',
)
return { viewsKey: params.viewsKey }
} catch (err) {
if (err instanceof ApplicationFailure) {
throw err
}
const errMsg = err instanceof Error ? err.message : String(err)
throw ApplicationFailure.create({
message: `Failed to generate 3D views: ${errMsg}`,
nonRetryable: true,
})
}
}