mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 19:07:41 +00:00
fix(transcode): preserve trashed status during background jobs and handle permanent purge (#416)
* fix(transcode): preserve trashed status during background jobs and handle permanent purge * fix(workflow): clear heartbeat and add info logs on task cancellation * fix(transcode): terminate media processes and guard task activities on cancellation - Pass AbortSignal to ffmpeg and pdftoppm child processes in transcodeService to terminate processes on task cancellation - Add local task AbortController context and cancellation guards in executeActivity to prevent subsequent workflow activities - Forward taskId and cancellation signals in transcode, screenshot, and PDF activities - Use updateMany instead of update in task activities to eliminate Prisma P2025 errors on purged task records * fix(workflow): allow cleanup activities in finally blocks during task cancellation Remove blanket cancellation check in executeActivity so that workflows can execute cleanup and failure handling activities in catch and finally blocks upon cancellation, while relying on AbortSignal in active activities to abort execution. * docs(changelog): improve user friendliness of changelog entries and formatting example * docs(agents): update changelog guidelines to emphasize end-user friendly descriptions * fix(transcode): handle queued cancellation and delete races in media workflow
This commit is contained in:
@@ -595,6 +595,83 @@ describe('Transcode Activities', () => {
|
||||
expect(updated?.hasJpegPreview).toBe(false)
|
||||
expect(s3Service.putObject).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge in updateAssetMediaActivity', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'purging.mp4',
|
||||
storageKey: { create: { key: 'purging.mp4' } },
|
||||
status: 'pending_purge',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
|
||||
await expect(
|
||||
updateAssetMediaActivity({
|
||||
assetId: asset.id,
|
||||
mediaInfo: { duration: 100 } as unknown as PrismaJson.MediaInfo,
|
||||
}),
|
||||
).rejects.toThrow('Asset not found, has no key, or is being purged')
|
||||
|
||||
expect(putObjectSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should tolerate missing or deleted asset without throwing P2025 in updateAssetMediaActivity', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'deleted-during-update.mp4',
|
||||
storageKey: { create: { key: 'deleted-during-update.mp4' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
putObjectSpy.mockImplementationOnce(async (): Promise<any> => {
|
||||
await prisma.asset.delete({ where: { id: asset.id } })
|
||||
return {}
|
||||
})
|
||||
|
||||
await expect(
|
||||
updateAssetMediaActivity({
|
||||
assetId: asset.id,
|
||||
mediaInfo: { duration: 100 } as unknown as PrismaJson.MediaInfo,
|
||||
}),
|
||||
).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge in transcodeVideoActivity', async () => {
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'purging-video.mp4',
|
||||
storageKey: { create: { key: 'purging-video.mp4' } },
|
||||
status: 'pending_purge',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
|
||||
await expect(
|
||||
transcodeVideoActivity({
|
||||
assetKey: 'purging-video.mp4',
|
||||
filePath: '/tmp/purging-video.mp4',
|
||||
videoSpec: { resolution: '720p', width: 1280, height: 720 },
|
||||
duration: 10,
|
||||
originalFps: 30,
|
||||
}),
|
||||
).rejects.toThrow('Asset or storage key has been purged or is pending purge')
|
||||
|
||||
expect(putObjectSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should call transcodeService.transcodeVideo', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
|
||||
@@ -733,6 +810,26 @@ describe('Transcode Activities', () => {
|
||||
).rejects.toThrowError(/Video transcoding failed/)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when transcodeVideoActivity is cancelled', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
const abortError = new Error('The operation was aborted')
|
||||
abortError.name = 'AbortError'
|
||||
vi.mocked(transcodeService.transcodeVideo).mockRejectedValue(abortError)
|
||||
|
||||
const promise = transcodeVideoActivity({
|
||||
assetKey: 'v.mp4',
|
||||
filePath: '/tmp/v.mp4',
|
||||
videoSpec: { resolution: '720p', width: 1280, height: 720 },
|
||||
duration: 10,
|
||||
originalFps: 30,
|
||||
})
|
||||
|
||||
await expect(promise).rejects.toThrowError(/Video transcoding cancelled/)
|
||||
await expect(promise).rejects.toMatchObject({
|
||||
nonRetryable: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when transcodeImageActivity fails with sharp error', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(transcodeService.transcodeImage).mockRejectedValue(new Error('sharp failed'))
|
||||
@@ -960,16 +1057,15 @@ describe('Transcode Activities', () => {
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
file: string,
|
||||
args: string[],
|
||||
cb: (err: Error | null, result: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
(file: string, args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
|
||||
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
|
||||
if (file === 'ffmpeg') {
|
||||
const outPath = args[args.length - 1]
|
||||
fs.writeFileSync(outPath, 'fake-poster-bytes')
|
||||
}
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
if (typeof cb === 'function') {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
@@ -992,6 +1088,7 @@ describe('Transcode Activities', () => {
|
||||
'-c:v',
|
||||
'libwebp',
|
||||
]),
|
||||
expect.objectContaining({ signal: undefined }),
|
||||
expect.any(Function),
|
||||
)
|
||||
expect(s3Service.putObject).toHaveBeenCalledWith(
|
||||
@@ -1003,6 +1100,96 @@ describe('Transcode Activities', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when poster extraction is cancelled with aborted signal', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
await expect(
|
||||
extractPosterActivity({
|
||||
assetKey: 'video.mp4',
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when poster extraction ffmpeg execution is aborted', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
|
||||
const controller = new AbortController()
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(_file: string, _args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
|
||||
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
|
||||
controller.abort()
|
||||
const abortErr = new Error('The operation was aborted')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(abortErr as any).code = 'ABORT_ERR'
|
||||
if (typeof cb === 'function') {
|
||||
cb(abortErr, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await expect(
|
||||
extractPosterActivity({
|
||||
assetKey: 'video.mp4',
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge before poster upload', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'purging-poster.mp4',
|
||||
storageKey: { create: { key: 'purging-poster.mp4' } },
|
||||
status: 'pending_purge',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(file: string, args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
|
||||
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
|
||||
if (file === 'ffmpeg') {
|
||||
const outPath = args[args.length - 1]
|
||||
fs.writeFileSync(outPath, 'fake-poster-bytes')
|
||||
}
|
||||
if (typeof cb === 'function') {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await expect(
|
||||
extractPosterActivity({
|
||||
assetKey: 'purging-poster.mp4',
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: 'Asset or storage key has been purged or is pending purge',
|
||||
nonRetryable: true,
|
||||
})
|
||||
|
||||
expect(putObjectSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should delete S3 object successfully', async () => {
|
||||
await deleteS3ObjectActivity({ key: 'files/test-key.mp4' })
|
||||
expect(s3Service.deleteObject).toHaveBeenCalledWith('shumai', 'files/test-key.mp4')
|
||||
@@ -1251,5 +1438,56 @@ describe('Transcode Activities', () => {
|
||||
})
|
||||
expect(task).toBeNull()
|
||||
})
|
||||
|
||||
it('should create ai_metadata_autofill task when asset is in trash (isDeleted: true)', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Test User Trashed Autofill', email: 'test-trashed-autofill@test.com' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team Trashed Autofill' },
|
||||
})
|
||||
await prisma.teamMember.create({
|
||||
data: { teamId: team.id, userId: user.id, role: 'owner' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Test Project Trashed Autofill', teamId: team.id },
|
||||
})
|
||||
const autofillAgent = await prisma.agent.create({
|
||||
data: {
|
||||
id: user.id,
|
||||
type: 'autofill',
|
||||
enabled: true,
|
||||
teamId: team.id,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
config: { provider: 'test', model: 'test' } as any,
|
||||
},
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'trashed-video.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
type: 'file',
|
||||
status: 'trashed',
|
||||
isDeleted: true,
|
||||
projectId: project.id,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
media: { proxyType: 'video' } as any,
|
||||
},
|
||||
})
|
||||
|
||||
await createAutofillTaskIfEnabledActivity({
|
||||
assetId: asset.id,
|
||||
teamId: team.id,
|
||||
projectId: project.id,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: 'ai_metadata_autofill' },
|
||||
})
|
||||
expect(task).toBeDefined()
|
||||
expect(task?.status).toBe('pending')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { AssetStatus, prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
@@ -12,7 +12,17 @@ import {
|
||||
isMarkdownDocument,
|
||||
isOfficeDocument,
|
||||
} from '@shumai/core/src/utils/mime'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { ApplicationFailure, Context } from '@temporalio/activity'
|
||||
import { getLocalTaskAbortSignal } from '@shumai/workflow-core'
|
||||
|
||||
function getActivityCancellationSignal(taskId?: string): AbortSignal | undefined {
|
||||
try {
|
||||
return Context.current().cancellationSignal
|
||||
} catch {
|
||||
return getLocalTaskAbortSignal(taskId)
|
||||
}
|
||||
}
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
@@ -47,6 +57,24 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureAssetNotPurging(assetKey: string): Promise<void> {
|
||||
const sk = await prisma.storageKey.findUnique({
|
||||
where: { key: assetKey },
|
||||
include: { assets: { select: { id: true, status: true } } },
|
||||
})
|
||||
if (
|
||||
sk &&
|
||||
(sk.status === 'purging' ||
|
||||
sk.assets.length === 0 ||
|
||||
sk.assets.every((a) => a.status === AssetStatus.pending_purge))
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset or storage key has been purged or is pending purge',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export async function getMediaInfoActivity(params: {
|
||||
filePath: string
|
||||
assetId: string
|
||||
@@ -216,6 +244,7 @@ export async function getMediaInfoActivity(params: {
|
||||
}
|
||||
|
||||
export interface VideoActivityParams {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
videoSpec: PrismaJson.VideoTranscode
|
||||
@@ -234,6 +263,14 @@ export async function transcodeVideoActivity(
|
||||
const res = params.videoSpec.resolution || `${params.videoSpec.height}p`
|
||||
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-${res}.mp4`)
|
||||
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Video transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, key)
|
||||
return { ...params.videoSpec, key }
|
||||
@@ -269,10 +306,12 @@ export async function transcodeVideoActivity(
|
||||
hardwareAcceleration: params.hardwareAcceleration,
|
||||
sourceVideoBitrate: params.sourceVideoBitrate,
|
||||
threads: params.threads,
|
||||
signal,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outputFile)
|
||||
const stream = Bun.file(outputFile).stream()
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
|
||||
|
||||
return { ...params.videoSpec, key }
|
||||
@@ -280,6 +319,19 @@ export async function transcodeVideoActivity(
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Video transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -302,6 +354,7 @@ export async function transcodeVideoActivity(
|
||||
}
|
||||
|
||||
export interface AudioActivityParams {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
threads?: number
|
||||
@@ -314,6 +367,14 @@ export async function transcodeAudioActivity(
|
||||
const stem = stemFromKey(params.assetKey)
|
||||
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
|
||||
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Audio transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, key)
|
||||
return { width: 0, height: 0, key }
|
||||
@@ -330,10 +391,12 @@ export async function transcodeAudioActivity(
|
||||
outputFile,
|
||||
bitrate: '128k',
|
||||
threads: params.threads,
|
||||
signal,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outputFile)
|
||||
const stream = Bun.file(outputFile).stream()
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
|
||||
|
||||
return { width: 0, height: 0, key }
|
||||
@@ -341,6 +404,19 @@ export async function transcodeAudioActivity(
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Audio transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -400,6 +476,7 @@ export async function transcodeImageActivity(
|
||||
},
|
||||
)
|
||||
const buffer = fs.readFileSync(outputFile)
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
|
||||
|
||||
return { ...params.imageSpec, key }
|
||||
@@ -430,6 +507,7 @@ export async function transcodeImageActivity(
|
||||
}
|
||||
|
||||
export interface GenerateSpriteActivityParams {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
mediaInfo: PrismaJson.MediaInfo
|
||||
@@ -439,6 +517,14 @@ export interface GenerateSpriteActivityParams {
|
||||
|
||||
export async function generateSpriteActivity(params: GenerateSpriteActivityParams) {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Sprite/Poster generation cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, params.spriteSpec.key)
|
||||
await s3Service.headObject(bucket, params.posterSpec.key)
|
||||
@@ -457,6 +543,7 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
signal,
|
||||
)
|
||||
if (params.mediaInfo && params.mediaInfo.metadata) {
|
||||
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
|
||||
@@ -470,10 +557,12 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
spriteFile,
|
||||
posterFile,
|
||||
params.mediaInfo.duration,
|
||||
signal,
|
||||
)
|
||||
}
|
||||
|
||||
const spriteBuffer = fs.readFileSync(spriteFile)
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(
|
||||
bucket,
|
||||
params.spriteSpec.key,
|
||||
@@ -496,6 +585,19 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Sprite/Poster generation cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -610,6 +712,7 @@ export async function generatePdfProxyActivity(
|
||||
|
||||
const stat = fs.statSync(pdfFilePath)
|
||||
const stream = Bun.file(pdfFilePath).stream()
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, pdfProxyKey, stream, stat.size, 'application/pdf')
|
||||
|
||||
return { pdfProxyKey, pdfFilePath }
|
||||
@@ -668,9 +771,9 @@ export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityP
|
||||
include: { storageKey: true },
|
||||
})
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) {
|
||||
if (!asset || !key || asset.status === AssetStatus.pending_purge) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset not found or has no key',
|
||||
message: 'Asset not found, has no key, or is being purged',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
@@ -679,14 +782,24 @@ export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityP
|
||||
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
|
||||
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
|
||||
|
||||
await prisma.asset.update({
|
||||
where: { id: params.assetId },
|
||||
const { count } = await prisma.asset.updateMany({
|
||||
where: {
|
||||
id: params.assetId,
|
||||
status: { not: AssetStatus.pending_purge },
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: { media: params.mediaInfo as any, hasJpegPreview: false },
|
||||
})
|
||||
if (count === 0) {
|
||||
logger.info(
|
||||
{ assetId: params.assetId },
|
||||
'[updateAssetMediaActivity] Asset was deleted or pending purge before media update completed',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export async function takeScreenshotsActivity(params: {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
assetId: string
|
||||
start: number
|
||||
@@ -695,11 +808,33 @@ export async function takeScreenshotsActivity(params: {
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
}): Promise<Array<{ key: string; timestamp: number }>> {
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Screenshot extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
return await transcodeService.takeScreenshots(params)
|
||||
return await transcodeService.takeScreenshots({ ...params, signal })
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Screenshot extraction cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -720,11 +855,21 @@ export async function takeScreenshotsActivity(params: {
|
||||
export interface ExtractPosterActivityParams {
|
||||
assetKey: string
|
||||
posterSpec: PrismaJson.PosterInfo
|
||||
taskId?: string
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export async function extractPosterActivity(
|
||||
params: ExtractPosterActivityParams,
|
||||
): Promise<{ poster: PrismaJson.PosterInfo }> {
|
||||
const signal = params.signal || getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
try {
|
||||
await s3Service.headObject(bucket, params.posterSpec.key)
|
||||
@@ -758,9 +903,10 @@ export async function extractPosterActivity(
|
||||
'75',
|
||||
posterFile,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
|
||||
|
||||
const posterBuffer = fs.readFileSync(posterFile)
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(
|
||||
bucket,
|
||||
params.posterSpec.key,
|
||||
@@ -773,6 +919,20 @@ export async function extractPosterActivity(
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -820,6 +980,7 @@ export async function overlayAnnotationsActivity(params: {
|
||||
}
|
||||
|
||||
export async function renderPdfPagesActivity(params: {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
assetId: string
|
||||
start: number
|
||||
@@ -827,11 +988,33 @@ export async function renderPdfPagesActivity(params: {
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
}): Promise<Array<{ key: string; page: number }>> {
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'PDF page rendering cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
return await transcodeService.renderPdfPages(params)
|
||||
return await transcodeService.renderPdfPages({ ...params, signal })
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'PDF page rendering cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -937,7 +1120,7 @@ export async function createAutofillTaskIfEnabledActivity(
|
||||
project: true,
|
||||
},
|
||||
})
|
||||
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack') || asset.isDeleted) {
|
||||
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack')) {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@ export async function fetchAssetWithKey(workerQueue: string, assetId: string) {
|
||||
const { getAssetActivity } = getActivities()
|
||||
const asset = await executeActivity(workerQueue, getAssetActivity, assetId)
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) {
|
||||
if (!asset || !key || asset.status === 'pending_purge') {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset not found or has no key',
|
||||
message: 'Asset not found, has no key, or is being purged',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ describe('renderPdfPagesWorkflow', () => {
|
||||
await renderPdfPagesWorkflow(task)
|
||||
|
||||
expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-pdf-pages',
|
||||
assetKey: 'doc.pdf',
|
||||
assetId: 'asset-pdf',
|
||||
start: 1,
|
||||
|
||||
@@ -21,6 +21,7 @@ export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void>
|
||||
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
|
||||
|
||||
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: pdfKey,
|
||||
assetId: asset.id,
|
||||
start: payload.pdfPages.start,
|
||||
|
||||
@@ -87,6 +87,7 @@ describe('takeVideoScreenshotsWorkflow', () => {
|
||||
await takeVideoScreenshotsWorkflow(task)
|
||||
|
||||
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-screenshot',
|
||||
assetKey: 'video.mp4',
|
||||
assetId: 'asset-video',
|
||||
start: 0,
|
||||
@@ -157,6 +158,7 @@ describe('takeVideoScreenshotsWorkflow', () => {
|
||||
await takeVideoScreenshotsWorkflow(task)
|
||||
|
||||
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-screenshot-proxy',
|
||||
assetKey: 'proxy-1080p.mp4',
|
||||
assetId: 'asset-video-proxy',
|
||||
start: 1.2345,
|
||||
|
||||
@@ -33,6 +33,7 @@ export async function takeVideoScreenshotsWorkflow(task: WorkflowTask): Promise<
|
||||
const { takeScreenshotsActivity } = getActivities()
|
||||
|
||||
const screenshots = await executeActivity(workerQueue, takeScreenshotsActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: targetKey,
|
||||
assetId: asset.id,
|
||||
start: payload.screenshot.start,
|
||||
|
||||
@@ -304,11 +304,13 @@ describe('transcodeVideoWorkflow', () => {
|
||||
|
||||
await transcodeVideoWorkflow(task)
|
||||
|
||||
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'audio.wav',
|
||||
filePath: '/tmp/video.mp4',
|
||||
threads: 4,
|
||||
})
|
||||
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
assetKey: 'audio.wav',
|
||||
filePath: '/tmp/video.mp4',
|
||||
threads: 4,
|
||||
}),
|
||||
)
|
||||
|
||||
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-audio',
|
||||
@@ -390,6 +392,7 @@ describe('transcodeVideoWorkflow', () => {
|
||||
posterSpec: {
|
||||
key: 'files/asset-poster/poster.webp',
|
||||
},
|
||||
taskId: 'task-poster',
|
||||
})
|
||||
|
||||
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
|
||||
|
||||
@@ -68,6 +68,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
const posterResult = await executeActivity(workerQueue, extractPosterActivity, {
|
||||
assetKey: key,
|
||||
posterSpec,
|
||||
taskId: task.id,
|
||||
})
|
||||
mediaInfo.poster = posterResult.poster
|
||||
await executeActivity(workerQueue, updateAssetMediaActivity, {
|
||||
@@ -98,6 +99,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
}
|
||||
|
||||
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
videoSpec,
|
||||
@@ -119,6 +121,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
const isAudio = mediaInfo.proxyType === 'audio'
|
||||
if (isAudio && metadata) {
|
||||
const audioTranscode = await executeActivity(workerQueue, transcodeAudioActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
threads: spec.threads,
|
||||
@@ -150,6 +153,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
}
|
||||
|
||||
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
spriteSpec,
|
||||
|
||||
Reference in New Issue
Block a user