fix(transcode): preserve trashed status during background jobs and handle permanent purge (#416)

* fix(transcode): preserve trashed status during background jobs and handle permanent purge

* fix(workflow): clear heartbeat and add info logs on task cancellation

* fix(transcode): terminate media processes and guard task activities on cancellation

- Pass AbortSignal to ffmpeg and pdftoppm child processes in transcodeService to terminate processes on task cancellation
- Add local task AbortController context and cancellation guards in executeActivity to prevent subsequent workflow activities
- Forward taskId and cancellation signals in transcode, screenshot, and PDF activities
- Use updateMany instead of update in task activities to eliminate Prisma P2025 errors on purged task records

* fix(workflow): allow cleanup activities in finally blocks during task cancellation

Remove blanket cancellation check in executeActivity so that workflows can execute cleanup and failure handling activities in catch and finally blocks upon cancellation, while relying on AbortSignal in active activities to abort execution.

* docs(changelog): improve user friendliness of changelog entries and formatting example

* docs(agents): update changelog guidelines to emphasize end-user friendly descriptions

* fix(transcode): handle queued cancellation and delete races in media workflow
This commit is contained in:
Yiling-J
2026-09-19 10:40:49 +08:00
committed by GitHub
parent 145aa6bfb4
commit b52b2ce7d8
25 changed files with 1172 additions and 65 deletions
+2 -2
View File
@@ -68,8 +68,8 @@ We maintain `CHANGELOG.md` following [Keep a Changelog](https://keepachangelog.c
Before opening a pull request, add an entry under `## [Unreleased]` for any user-facing changes:
- **Focus on User Impact**: Describe the user-facing outcome or observable behavior in plain terms, not internal code mechanics, file paths, or private symbols.
- **Formatting**: `- **<scope>**: <Description>` (e.g. `- **comment**: Add emoji reaction support for comments`).
- **End-User Friendly**: Make entries easy to understand for end users. For new features, explain what they actually do in plain language without heavy technical jargon. For bug fixes, describe what the bug was in a friendly, relatable way rather than focusing on internal code mechanics, file paths, or private symbols.
- **Formatting**: `- **<scope>**: <Description>` (e.g. `- **asset**: Fix an issue where moving a video to trash while it was still transcoding could cause it to lose its trashed status`).
- **No PR Numbers**: Do not include PR numbers or links; the release pipeline automatically appends the detailed PR list.
- **Internal Changes**: Routine refactors, test additions, or internal chores without external behavior changes may omit changelog updates.
+2
View File
@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Fixed
- **asset**: Fix an issue where moving a video to trash while it was still transcoding could cause it to lose its trashed status
- **transcode**: Immediately stop background video processing and transcoding when a file is permanently deleted from the trash
- **asset**: Fix proxy download filename using original file extension instead of proxy extension
- **webui**: Center breadcrumb file action popup menu on the chevron icon
+145 -1
View File
@@ -3,13 +3,14 @@ import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { HTTPException } from 'hono/http-exception'
import { AssetType, AssetStatus, Prisma } from '@shumai/db'
import { AssetType, AssetStatus, Prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { uploadService } from '@shumai/core/src/upload/upload'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { AssetService } from './asset'
import { ensureJpegInStorage } from '@shumai/core/src/s3/preview-converter'
import { workflowService } from '@shumai/workflow-core'
vi.mock('@shumai/core/src/s3/s3', () => ({
s3Service: {
@@ -3004,6 +3005,65 @@ describe('AssetService — natural sort by name', () => {
await expect(assetService.emptyTrash(project.id)).resolves.not.toThrow()
})
it('should purge trashed assets whose status was processed, cancel their in-flight tasks, and delete WorkflowTasks', async () => {
const cancelSpy = vi.spyOn(workflowService, 'cancel').mockResolvedValue(undefined)
const team = await prisma.team.create({ data: { name: 'Purge Task Team' } })
const project = await prisma.project.create({
data: { name: 'Purge Task Project', teamId: team.id },
})
const rootFolder = await prisma.asset.create({
data: {
name: 'root',
type: AssetType.folder,
projectId: project.id,
status: AssetStatus.uploaded,
},
})
const user = await prisma.user.create({
data: { name: 'Test User Purge', email: `test-purge-${Date.now()}@example.com` },
})
// Asset was deleted, but status was set to 'processed' by background activity
const file = await prisma.asset.create({
data: {
name: 'skewed-status.mp4',
type: AssetType.file,
status: AssetStatus.processed,
isDeleted: true,
deletedAt: new Date(),
project: { connect: { id: project.id } },
parent: { connect: { id: rootFolder.id } },
creator: { connect: { id: user.id } },
storageKey: { create: { key: 'files/skewed-status/raw.mp4' } },
},
})
// In-flight task for this asset
const task = await prisma.workflowTask.create({
data: {
assetId: file.id,
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.processing,
},
})
await assetService.emptyTrash(project.id)
// Asset should be deleted from DB
const assetInDb = await prisma.asset.findUnique({ where: { id: file.id } })
expect(assetInDb).toBeNull()
// Task should be cancelled
expect(cancelSpy).toHaveBeenCalledWith(task.id)
// WorkflowTask should be deleted from DB
const taskInDb = await prisma.workflowTask.findUnique({ where: { id: task.id } })
expect(taskInDb).toBeNull()
cancelSpy.mockRestore()
})
})
describe('expireTrashedAssets', () => {
@@ -4904,4 +4964,88 @@ describe('AssetService — natural sort by name', () => {
expect(stackInfo?.commentsCount).toBe(1)
})
})
describe('restoreAssets with in-flight transcoding', () => {
it('should restore asset with status "processing" if a transcode task is still active', async () => {
const team = await prisma.team.create({ data: { name: 'Restore Transcode Team' } })
const project = await prisma.project.create({
data: { name: 'Restore Transcode Project', teamId: team.id },
})
const rootFolder = await prisma.asset.create({
data: {
name: 'root',
type: AssetType.folder,
projectId: project.id,
status: AssetStatus.uploaded,
},
})
const file = await prisma.asset.create({
data: {
name: 'transcoding-video.mp4',
type: AssetType.file,
projectId: project.id,
parentId: rootFolder.id,
status: AssetStatus.trashed,
isDeleted: true,
deletedAt: new Date(),
},
})
// Create an active transcode task
await prisma.workflowTask.create({
data: {
assetId: file.id,
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.processing,
},
})
await assetService.restoreAssets([file.id])
const restored = await prisma.asset.findUnique({ where: { id: file.id } })
expect(restored?.isDeleted).toBe(false)
expect(restored?.status).toBe(AssetStatus.processing)
})
it('should restore asset with status "processed" if no active transcode task exists', async () => {
const team = await prisma.team.create({ data: { name: 'Restore Completed Team' } })
const project = await prisma.project.create({
data: { name: 'Restore Completed Project', teamId: team.id },
})
const rootFolder = await prisma.asset.create({
data: {
name: 'root',
type: AssetType.folder,
projectId: project.id,
status: AssetStatus.uploaded,
},
})
const file = await prisma.asset.create({
data: {
name: 'completed-video.mp4',
type: AssetType.file,
projectId: project.id,
parentId: rootFolder.id,
status: AssetStatus.trashed,
isDeleted: true,
deletedAt: new Date(),
},
})
// Completed transcode task
await prisma.workflowTask.create({
data: {
assetId: file.id,
type: WorkflowTaskType.transcode_video,
status: WorkflowTaskStatus.completed,
},
})
await assetService.restoreAssets([file.id])
const restored = await prisma.asset.findUnique({ where: { id: file.id } })
expect(restored?.isDeleted).toBe(false)
expect(restored?.status).toBe(AssetStatus.processed)
})
})
})
+84 -8
View File
@@ -1,4 +1,4 @@
import { prisma } from '@shumai/db'
import { cancelWorkflowTask, prisma } from '@shumai/db'
import {
AncestorFolder,
AssetInfo,
@@ -28,7 +28,15 @@ import {
type ExportOptions,
type ExportResult,
} from './comments-export'
import { type Asset, AssetStatus, AssetType, Prisma, type StorageKey } from '@shumai/db'
import {
type Asset,
AssetStatus,
AssetType,
Prisma,
type StorageKey,
WorkflowTaskStatus,
WorkflowTaskType,
} from '@shumai/db'
import { HTTPException } from 'hono/http-exception'
import { logger } from '@shumai/core/src/logger'
import { PaginatedData, paginateQuery, PaginationParams } from '@shumai/core/src/pagination'
@@ -1343,7 +1351,7 @@ export class AssetService {
private async cascadeStatusToPendingPurge(rootIds: string[]): Promise<void> {
if (rootIds.length === 0) return
await this.prismaClient.$executeRaw`
const rows = await this.prismaClient.$queryRaw<{ id: string }[]>`
WITH RECURSIVE descendant AS (
SELECT id FROM assets WHERE id = ANY(${rootIds})
UNION ALL
@@ -1358,15 +1366,35 @@ export class AssetService {
WHERE s.type = 'symlink' AND s.target_id IN (SELECT id FROM all_purging_targets)
)
UPDATE assets SET status = 'pending_purge', updated_at = NOW()
WHERE id IN (SELECT id FROM all_purging_targets) OR id IN (SELECT id FROM all_affected_symlinks);
WHERE id IN (SELECT id FROM all_purging_targets) OR id IN (SELECT id FROM all_affected_symlinks)
RETURNING id;
`
const affectedIds = rows.map((r) => r.id)
if (affectedIds.length > 0) {
const inFlightTasks = await this.prismaClient.workflowTask.findMany({
where: {
assetId: { in: affectedIds },
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
select: { id: true },
})
for (const task of inFlightTasks) {
try {
logger.info({ taskId: task.id }, 'Cancelling in-flight workflow task for purging asset')
await cancelWorkflowTask(task.id)
} catch (err) {
logger.warn({ taskId: task.id, err }, 'Failed to cancel workflow task during purge')
}
}
}
}
async emptyTrash(projectId: string): Promise<void> {
const trashedRoots = await this.prismaClient.asset.findMany({
where: {
projectId,
status: 'trashed',
isDeleted: true,
},
select: { id: true },
@@ -1418,9 +1446,9 @@ export class AssetService {
// Find up to 100 expired roots, ordered by oldest first to avoid starvation
const expiredRoots = await this.prismaClient.asset.findMany({
where: {
status: 'trashed',
deletedAt: { lt: thirtyDaysAgo },
isDeleted: true,
status: { not: AssetStatus.pending_purge },
deletedAt: { lt: thirtyDaysAgo },
},
orderBy: { deletedAt: 'asc' },
select: { id: true },
@@ -1435,6 +1463,35 @@ export class AssetService {
* Stage 2: Delete DB records for assets marked as 'pending_purge'.
*/
private async purgePendingAssets() {
const pendingAssets = await this.prismaClient.asset.findMany({
where: { status: AssetStatus.pending_purge },
select: { id: true },
})
const pendingAssetIds = pendingAssets.map((a) => a.id)
if (pendingAssetIds.length > 0) {
const inFlightTasks = await this.prismaClient.workflowTask.findMany({
where: {
assetId: { in: pendingAssetIds },
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
},
select: { id: true },
})
for (const task of inFlightTasks) {
try {
logger.info({ taskId: task.id }, 'Cancelling in-flight workflow task before purge')
await cancelWorkflowTask(task.id)
} catch (err) {
logger.warn({ taskId: task.id, err }, 'Failed to cancel workflow task during purge')
}
}
await this.prismaClient.workflowTask.deleteMany({
where: { assetId: { in: pendingAssetIds } },
})
}
const { count } = await this.prismaClient.asset.deleteMany({
where: { status: AssetStatus.pending_purge },
})
@@ -1577,11 +1634,30 @@ export class AssetService {
}
}
const activeTask = await tx.workflowTask.findFirst({
where: {
assetId: a.id,
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
type: {
in: [
WorkflowTaskType.transcode,
WorkflowTaskType.transcode_video,
WorkflowTaskType.transcode_image,
WorkflowTaskType.transcode_pdf,
WorkflowTaskType.transcode_pdf_pages,
WorkflowTaskType.transcode_screenshot,
WorkflowTaskType.transcode_image_annotation,
WorkflowTaskType.transcode_watermark,
],
},
},
})
await tx.asset.update({
where: { id: a.id },
data: {
isDeleted: false,
status: 'processed',
status: activeTask ? AssetStatus.processing : AssetStatus.processed,
deletedAt: null,
},
})
@@ -1776,5 +1776,41 @@ describe('TranscodeService', () => {
expect.any(Function),
)
})
it('transcodeVideo with signal should pass signal to execFile', async () => {
vi.mocked(execFile).mockImplementation(
(
_cmd: unknown,
_args: unknown,
optionsOrCallback: unknown,
maybeCallback?: unknown,
): ReturnType<typeof child_process.execFile> => {
const cb = (
typeof optionsOrCallback === 'function' ? optionsOrCallback : maybeCallback
) as (err: Error | null, result: { stdout: string; stderr: string }) => void
if (typeof cb === 'function') {
cb(null, { stdout: '', stderr: '' })
}
return {} as ReturnType<typeof child_process.execFile>
},
)
const controller = new AbortController()
const outputFile = path.join(tempDir, 'out_signal.mp4')
await transcodeService.transcodeVideo({
inputFile: 'input.mp4',
outputFile,
width: 1280,
height: 720,
signal: controller.signal,
})
expect(child_process.execFile).toHaveBeenCalledWith(
'ffmpeg',
expect.any(Array),
{ signal: controller.signal },
expect.any(Function),
)
})
})
})
+51 -8
View File
@@ -117,6 +117,7 @@ export interface TranscodeVideoParams {
videoBitrate?: string
sourceVideoBitrate?: number
threads?: number
signal?: AbortSignal
}
export interface EncoderConfig {
@@ -634,7 +635,13 @@ export class TranscodeService {
args.push('-movflags', '+faststart', '-max_muxing_queue_size', '1024', params.outputFile)
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
if (params.signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
signal: params.signal,
})
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
}
async transcodeImage(
@@ -741,6 +748,7 @@ export class TranscodeService {
outputSprite: string,
outputPoster: string,
duration: number,
signal?: AbortSignal,
): Promise<void> {
const spriteFps = 100 / duration
const filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out];[0:v]scale=-2:300:force_original_aspect_ratio=decrease,select='eq(n\\,0)'[thumb_out]`
@@ -771,19 +779,30 @@ export class TranscodeService {
'1024',
outputPoster,
]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
if (signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
}
async generatePdfSprite(
inputFile: string,
outputSprite: string,
outputPoster: string,
signal?: AbortSignal,
): Promise<{ pageCount: number; originalWidth: number; originalHeight: number }> {
const tmpDir = this.createTempDir('pdf-sprite-')
try {
const pagePrefix = path.join(tmpDir, 'page')
try {
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix])
if (signal) {
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix], {
signal,
})
} else {
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix])
}
} catch (err) {
const errCode = (err as Record<string, unknown>)?.code
const msg = err instanceof Error ? err.message : String(err)
@@ -840,7 +859,11 @@ export class TranscodeService {
'75',
outputSprite,
]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs])
if (signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs], { signal })
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs])
}
await sharp(firstPagePath, { limitInputPixels: false })
.toColorspace('srgb')
@@ -1090,6 +1113,7 @@ export class TranscodeService {
outputFile: string
bitrate?: string
threads?: number
signal?: AbortSignal
}): Promise<void> {
const bitrate = params.bitrate || '128k'
const args = ['-i', params.inputFile, '-vn', '-c:a', 'aac', '-b:a', bitrate, '-ac', '2']
@@ -1097,7 +1121,13 @@ export class TranscodeService {
args.push('-threads', params.threads.toString())
}
args.push(params.outputFile)
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
if (params.signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
signal: params.signal,
})
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
}
async extractVideoFrames(params: ExtractVideoFramesParams): Promise<string[]> {
@@ -1198,6 +1228,7 @@ export class TranscodeService {
count: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
signal?: AbortSignal
}): Promise<Array<{ key: string; timestamp: number }>> {
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = this.createTempDir('screenshot-')
@@ -1259,7 +1290,13 @@ export class TranscodeService {
'80',
localShotPath,
]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
if (params.signal) {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
signal: params.signal,
})
} else {
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
}
const isMatch =
commentTimestamp !== undefined &&
@@ -1295,6 +1332,7 @@ export class TranscodeService {
end: number
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
signal?: AbortSignal
}): Promise<Array<{ key: string; page: number }>> {
const bucket = process.env.S3_BUCKET || 'shumai'
const tmpDir = this.createTempDir('pdf-pages-')
@@ -1307,7 +1345,7 @@ export class TranscodeService {
// 2. Render pages via pdftoppm (-png -f start -l end)
const pagePrefix = path.join(tmpDir, 'page')
try {
await execFileAsync('pdftoppm', [
const pdftoppmArgs = [
'-png',
'-f',
params.start.toString(),
@@ -1315,7 +1353,12 @@ export class TranscodeService {
params.end.toString(),
pdfPath,
pagePrefix,
])
]
if (params.signal) {
await execFileAsync('pdftoppm', pdftoppmArgs, { signal: params.signal })
} else {
await execFileAsync('pdftoppm', pdftoppmArgs)
}
} catch (err) {
const errCode = (err as Record<string, unknown>)?.code
const msg = err instanceof Error ? err.message : String(err)
+14
View File
@@ -18,6 +18,20 @@ export function registerWorkflowTrigger(cb: WorkflowTriggerCallback) {
onWorkflowTaskCreated = cb
}
// Global callback for workflow task cancellation (decouples db/core packages from workflow engine)
type WorkflowCancelCallback = (taskId: string) => Promise<void>
let onWorkflowTaskCancelled: WorkflowCancelCallback | null = null
export function registerWorkflowCancel(cb: WorkflowCancelCallback) {
onWorkflowTaskCancelled = cb
}
export async function cancelWorkflowTask(taskId: string): Promise<void> {
if (onWorkflowTaskCancelled) {
await onWorkflowTaskCancelled(taskId)
}
}
const isTest = process.env.NODE_ENV === 'test'
const globalForPrisma = globalThis as unknown as {
@@ -595,6 +595,83 @@ describe('Transcode Activities', () => {
expect(updated?.hasJpegPreview).toBe(false)
expect(s3Service.putObject).toHaveBeenCalled()
})
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge in updateAssetMediaActivity', async () => {
const asset = await prisma.asset.create({
data: {
name: 'purging.mp4',
storageKey: { create: { key: 'purging.mp4' } },
status: 'pending_purge',
type: 'file',
},
})
const putObjectSpy = vi.mocked(s3Service.putObject)
putObjectSpy.mockClear()
await expect(
updateAssetMediaActivity({
assetId: asset.id,
mediaInfo: { duration: 100 } as unknown as PrismaJson.MediaInfo,
}),
).rejects.toThrow('Asset not found, has no key, or is being purged')
expect(putObjectSpy).not.toHaveBeenCalled()
})
it('should tolerate missing or deleted asset without throwing P2025 in updateAssetMediaActivity', async () => {
const asset = await prisma.asset.create({
data: {
name: 'deleted-during-update.mp4',
storageKey: { create: { key: 'deleted-during-update.mp4' } },
status: 'uploaded',
type: 'file',
},
})
const putObjectSpy = vi.mocked(s3Service.putObject)
putObjectSpy.mockClear()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
putObjectSpy.mockImplementationOnce(async (): Promise<any> => {
await prisma.asset.delete({ where: { id: asset.id } })
return {}
})
await expect(
updateAssetMediaActivity({
assetId: asset.id,
mediaInfo: { duration: 100 } as unknown as PrismaJson.MediaInfo,
}),
).resolves.toBeUndefined()
})
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge in transcodeVideoActivity', async () => {
await prisma.asset.create({
data: {
name: 'purging-video.mp4',
storageKey: { create: { key: 'purging-video.mp4' } },
status: 'pending_purge',
type: 'file',
},
})
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
const putObjectSpy = vi.mocked(s3Service.putObject)
putObjectSpy.mockClear()
await expect(
transcodeVideoActivity({
assetKey: 'purging-video.mp4',
filePath: '/tmp/purging-video.mp4',
videoSpec: { resolution: '720p', width: 1280, height: 720 },
duration: 10,
originalFps: 30,
}),
).rejects.toThrow('Asset or storage key has been purged or is pending purge')
expect(putObjectSpy).not.toHaveBeenCalled()
})
it('should call transcodeService.transcodeVideo', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
@@ -733,6 +810,26 @@ describe('Transcode Activities', () => {
).rejects.toThrowError(/Video transcoding failed/)
})
it('should throw non-retryable ApplicationFailure when transcodeVideoActivity is cancelled', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
const abortError = new Error('The operation was aborted')
abortError.name = 'AbortError'
vi.mocked(transcodeService.transcodeVideo).mockRejectedValue(abortError)
const promise = transcodeVideoActivity({
assetKey: 'v.mp4',
filePath: '/tmp/v.mp4',
videoSpec: { resolution: '720p', width: 1280, height: 720 },
duration: 10,
originalFps: 30,
})
await expect(promise).rejects.toThrowError(/Video transcoding cancelled/)
await expect(promise).rejects.toMatchObject({
nonRetryable: true,
})
})
it('should throw non-retryable ApplicationFailure when transcodeImageActivity fails with sharp error', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
vi.mocked(transcodeService.transcodeImage).mockRejectedValue(new Error('sharp failed'))
@@ -960,16 +1057,15 @@ describe('Transcode Activities', () => {
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(
file: string,
args: string[],
cb: (err: Error | null, result: { stdout: string; stderr: string }) => void,
) => {
(file: string, args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
if (file === 'ffmpeg') {
const outPath = args[args.length - 1]
fs.writeFileSync(outPath, 'fake-poster-bytes')
}
cb(null, { stdout: '', stderr: '' })
if (typeof cb === 'function') {
cb(null, { stdout: '', stderr: '' })
}
},
)
@@ -992,6 +1088,7 @@ describe('Transcode Activities', () => {
'-c:v',
'libwebp',
]),
expect.objectContaining({ signal: undefined }),
expect.any(Function),
)
expect(s3Service.putObject).toHaveBeenCalledWith(
@@ -1003,6 +1100,96 @@ describe('Transcode Activities', () => {
)
})
it('should throw non-retryable ApplicationFailure when poster extraction is cancelled with aborted signal', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
const controller = new AbortController()
controller.abort()
await expect(
extractPosterActivity({
assetKey: 'video.mp4',
posterSpec: { key: 'files/asset-1/poster.webp' },
signal: controller.signal,
}),
).rejects.toMatchObject({
message: 'Poster extraction cancelled',
nonRetryable: true,
})
})
it('should throw non-retryable ApplicationFailure when poster extraction ffmpeg execution is aborted', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
const controller = new AbortController()
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(_file: string, _args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
controller.abort()
const abortErr = new Error('The operation was aborted')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(abortErr as any).code = 'ABORT_ERR'
if (typeof cb === 'function') {
cb(abortErr, { stdout: '', stderr: '' })
}
},
)
await expect(
extractPosterActivity({
assetKey: 'video.mp4',
posterSpec: { key: 'files/asset-1/poster.webp' },
signal: controller.signal,
}),
).rejects.toMatchObject({
message: 'Poster extraction cancelled',
nonRetryable: true,
})
})
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge before poster upload', async () => {
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
const putObjectSpy = vi.mocked(s3Service.putObject)
putObjectSpy.mockClear()
await prisma.asset.create({
data: {
name: 'purging-poster.mp4',
storageKey: { create: { key: 'purging-poster.mp4' } },
status: 'pending_purge',
type: 'file',
},
})
// eslint-disable-next-line @typescript-eslint/no-explicit-any
;(child_process.execFile as any).mockImplementation(
(file: string, args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
if (file === 'ffmpeg') {
const outPath = args[args.length - 1]
fs.writeFileSync(outPath, 'fake-poster-bytes')
}
if (typeof cb === 'function') {
cb(null, { stdout: '', stderr: '' })
}
},
)
await expect(
extractPosterActivity({
assetKey: 'purging-poster.mp4',
posterSpec: { key: 'files/asset-1/poster.webp' },
}),
).rejects.toMatchObject({
message: 'Asset or storage key has been purged or is pending purge',
nonRetryable: true,
})
expect(putObjectSpy).not.toHaveBeenCalled()
})
it('should delete S3 object successfully', async () => {
await deleteS3ObjectActivity({ key: 'files/test-key.mp4' })
expect(s3Service.deleteObject).toHaveBeenCalledWith('shumai', 'files/test-key.mp4')
@@ -1251,5 +1438,56 @@ describe('Transcode Activities', () => {
})
expect(task).toBeNull()
})
it('should create ai_metadata_autofill task when asset is in trash (isDeleted: true)', async () => {
const user = await prisma.user.create({
data: { name: 'Test User Trashed Autofill', email: 'test-trashed-autofill@test.com' },
})
const team = await prisma.team.create({
data: { name: 'Test Team Trashed Autofill' },
})
await prisma.teamMember.create({
data: { teamId: team.id, userId: user.id, role: 'owner' },
})
const project = await prisma.project.create({
data: { name: 'Test Project Trashed Autofill', teamId: team.id },
})
const autofillAgent = await prisma.agent.create({
data: {
id: user.id,
type: 'autofill',
enabled: true,
teamId: team.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
config: { provider: 'test', model: 'test' } as any,
},
})
const asset = await prisma.asset.create({
data: {
name: 'trashed-video.mp4',
mediaType: 'video/mp4',
type: 'file',
status: 'trashed',
isDeleted: true,
projectId: project.id,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
media: { proxyType: 'video' } as any,
},
})
await createAutofillTaskIfEnabledActivity({
assetId: asset.id,
teamId: team.id,
projectId: project.id,
})
const task = await prisma.workflowTask.findFirst({
where: { assetId: asset.id, type: 'ai_metadata_autofill' },
})
expect(task).toBeDefined()
expect(task?.status).toBe('pending')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
})
})
})
+193 -10
View File
@@ -1,4 +1,4 @@
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { AssetStatus, prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core/src/transcode/transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
@@ -12,7 +12,17 @@ import {
isMarkdownDocument,
isOfficeDocument,
} from '@shumai/core/src/utils/mime'
import { ApplicationFailure } from '@temporalio/activity'
import { logger } from '@shumai/core/src/logger'
import { ApplicationFailure, Context } from '@temporalio/activity'
import { getLocalTaskAbortSignal } from '@shumai/workflow-core'
function getActivityCancellationSignal(taskId?: string): AbortSignal | undefined {
try {
return Context.current().cancellationSignal
} catch {
return getLocalTaskAbortSignal(taskId)
}
}
import * as path from 'path'
import * as fs from 'fs'
import * as os from 'os'
@@ -47,6 +57,24 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
}
}
async function ensureAssetNotPurging(assetKey: string): Promise<void> {
const sk = await prisma.storageKey.findUnique({
where: { key: assetKey },
include: { assets: { select: { id: true, status: true } } },
})
if (
sk &&
(sk.status === 'purging' ||
sk.assets.length === 0 ||
sk.assets.every((a) => a.status === AssetStatus.pending_purge))
) {
throw ApplicationFailure.create({
message: 'Asset or storage key has been purged or is pending purge',
nonRetryable: true,
})
}
}
export async function getMediaInfoActivity(params: {
filePath: string
assetId: string
@@ -216,6 +244,7 @@ export async function getMediaInfoActivity(params: {
}
export interface VideoActivityParams {
taskId?: string
assetKey: string
filePath: string
videoSpec: PrismaJson.VideoTranscode
@@ -234,6 +263,14 @@ export async function transcodeVideoActivity(
const res = params.videoSpec.resolution || `${params.videoSpec.height}p`
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-${res}.mp4`)
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Video transcoding cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, key)
return { ...params.videoSpec, key }
@@ -269,10 +306,12 @@ export async function transcodeVideoActivity(
hardwareAcceleration: params.hardwareAcceleration,
sourceVideoBitrate: params.sourceVideoBitrate,
threads: params.threads,
signal,
})
const stat = fs.statSync(outputFile)
const stream = Bun.file(outputFile).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
return { ...params.videoSpec, key }
@@ -280,6 +319,19 @@ export async function transcodeVideoActivity(
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Video transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
@@ -302,6 +354,7 @@ export async function transcodeVideoActivity(
}
export interface AudioActivityParams {
taskId?: string
assetKey: string
filePath: string
threads?: number
@@ -314,6 +367,14 @@ export async function transcodeAudioActivity(
const stem = stemFromKey(params.assetKey)
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Audio transcoding cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, key)
return { width: 0, height: 0, key }
@@ -330,10 +391,12 @@ export async function transcodeAudioActivity(
outputFile,
bitrate: '128k',
threads: params.threads,
signal,
})
const stat = fs.statSync(outputFile)
const stream = Bun.file(outputFile).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
return { width: 0, height: 0, key }
@@ -341,6 +404,19 @@ export async function transcodeAudioActivity(
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Audio transcoding cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
@@ -400,6 +476,7 @@ export async function transcodeImageActivity(
},
)
const buffer = fs.readFileSync(outputFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
return { ...params.imageSpec, key }
@@ -430,6 +507,7 @@ export async function transcodeImageActivity(
}
export interface GenerateSpriteActivityParams {
taskId?: string
assetKey: string
filePath: string
mediaInfo: PrismaJson.MediaInfo
@@ -439,6 +517,14 @@ export interface GenerateSpriteActivityParams {
export async function generateSpriteActivity(params: GenerateSpriteActivityParams) {
const bucket = process.env.S3_BUCKET || 'shumai'
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Sprite/Poster generation cancelled',
nonRetryable: true,
})
}
try {
await s3Service.headObject(bucket, params.spriteSpec.key)
await s3Service.headObject(bucket, params.posterSpec.key)
@@ -457,6 +543,7 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
params.filePath,
spriteFile,
posterFile,
signal,
)
if (params.mediaInfo && params.mediaInfo.metadata) {
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
@@ -470,10 +557,12 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
spriteFile,
posterFile,
params.mediaInfo.duration,
signal,
)
}
const spriteBuffer = fs.readFileSync(spriteFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.spriteSpec.key,
@@ -496,6 +585,19 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Sprite/Poster generation cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
@@ -610,6 +712,7 @@ export async function generatePdfProxyActivity(
const stat = fs.statSync(pdfFilePath)
const stream = Bun.file(pdfFilePath).stream()
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(bucket, pdfProxyKey, stream, stat.size, 'application/pdf')
return { pdfProxyKey, pdfFilePath }
@@ -668,9 +771,9 @@ export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityP
include: { storageKey: true },
})
const key = asset?.storageKey?.key
if (!asset || !key) {
if (!asset || !key || asset.status === AssetStatus.pending_purge) {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
message: 'Asset not found, has no key, or is being purged',
nonRetryable: true,
})
}
@@ -679,14 +782,24 @@ export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityP
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
await prisma.asset.update({
where: { id: params.assetId },
const { count } = await prisma.asset.updateMany({
where: {
id: params.assetId,
status: { not: AssetStatus.pending_purge },
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
data: { media: params.mediaInfo as any, hasJpegPreview: false },
})
if (count === 0) {
logger.info(
{ assetId: params.assetId },
'[updateAssetMediaActivity] Asset was deleted or pending purge before media update completed',
)
}
}
export async function takeScreenshotsActivity(params: {
taskId?: string
assetKey: string
assetId: string
start: number
@@ -695,11 +808,33 @@ export async function takeScreenshotsActivity(params: {
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; timestamp: number }>> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Screenshot extraction cancelled',
nonRetryable: true,
})
}
try {
return await transcodeService.takeScreenshots(params)
return await transcodeService.takeScreenshots({ ...params, signal })
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Screenshot extraction cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
@@ -720,11 +855,21 @@ export async function takeScreenshotsActivity(params: {
export interface ExtractPosterActivityParams {
assetKey: string
posterSpec: PrismaJson.PosterInfo
taskId?: string
signal?: AbortSignal
}
export async function extractPosterActivity(
params: ExtractPosterActivityParams,
): Promise<{ poster: PrismaJson.PosterInfo }> {
const signal = params.signal || getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'Poster extraction cancelled',
nonRetryable: true,
})
}
const bucket = process.env.S3_BUCKET || 'shumai'
try {
await s3Service.headObject(bucket, params.posterSpec.key)
@@ -758,9 +903,10 @@ export async function extractPosterActivity(
'75',
posterFile,
]
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
const posterBuffer = fs.readFileSync(posterFile)
await ensureAssetNotPurging(params.assetKey)
await s3Service.putObject(
bucket,
params.posterSpec.key,
@@ -773,6 +919,20 @@ export async function extractPosterActivity(
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'Poster extraction cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
@@ -820,6 +980,7 @@ export async function overlayAnnotationsActivity(params: {
}
export async function renderPdfPagesActivity(params: {
taskId?: string
assetKey: string
assetId: string
start: number
@@ -827,11 +988,33 @@ export async function renderPdfPagesActivity(params: {
commentTimestamp?: number | null
annotations?: PrismaJson.AnnotationList | null
}): Promise<Array<{ key: string; page: number }>> {
const signal = getActivityCancellationSignal(params.taskId)
if (signal?.aborted) {
throw ApplicationFailure.create({
message: 'PDF page rendering cancelled',
nonRetryable: true,
})
}
try {
return await transcodeService.renderPdfPages(params)
return await transcodeService.renderPdfPages({ ...params, signal })
} catch (err) {
const { code, message } = getErrorDetails(err)
const lowerMsg = message.toLowerCase()
if (
signal?.aborted ||
code === 'ABORT_ERR' ||
message.includes('aborted') ||
lowerMsg.includes('abort')
) {
throw ApplicationFailure.create({
message: 'PDF page rendering cancelled',
nonRetryable: true,
cause: err instanceof Error ? err : undefined,
})
}
if (
code === 'ENOENT' ||
lowerMsg.includes('enoent') ||
@@ -937,7 +1120,7 @@ export async function createAutofillTaskIfEnabledActivity(
project: true,
},
})
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack') || asset.isDeleted) {
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack')) {
return
}
+2 -2
View File
@@ -16,9 +16,9 @@ export async function fetchAssetWithKey(workerQueue: string, assetId: string) {
const { getAssetActivity } = getActivities()
const asset = await executeActivity(workerQueue, getAssetActivity, assetId)
const key = asset?.storageKey?.key
if (!asset || !key) {
if (!asset || !key || asset.status === 'pending_purge') {
throw ApplicationFailure.create({
message: 'Asset not found or has no key',
message: 'Asset not found, has no key, or is being purged',
nonRetryable: true,
})
}
@@ -88,6 +88,7 @@ describe('renderPdfPagesWorkflow', () => {
await renderPdfPagesWorkflow(task)
expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalledWith({
taskId: 'task-pdf-pages',
assetKey: 'doc.pdf',
assetId: 'asset-pdf',
start: 1,
@@ -21,6 +21,7 @@ export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void>
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
taskId: task.id,
assetKey: pdfKey,
assetId: asset.id,
start: payload.pdfPages.start,
@@ -87,6 +87,7 @@ describe('takeVideoScreenshotsWorkflow', () => {
await takeVideoScreenshotsWorkflow(task)
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
taskId: 'task-screenshot',
assetKey: 'video.mp4',
assetId: 'asset-video',
start: 0,
@@ -157,6 +158,7 @@ describe('takeVideoScreenshotsWorkflow', () => {
await takeVideoScreenshotsWorkflow(task)
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
taskId: 'task-screenshot-proxy',
assetKey: 'proxy-1080p.mp4',
assetId: 'asset-video-proxy',
start: 1.2345,
@@ -33,6 +33,7 @@ export async function takeVideoScreenshotsWorkflow(task: WorkflowTask): Promise<
const { takeScreenshotsActivity } = getActivities()
const screenshots = await executeActivity(workerQueue, takeScreenshotsActivity, {
taskId: task.id,
assetKey: targetKey,
assetId: asset.id,
start: payload.screenshot.start,
@@ -304,11 +304,13 @@ describe('transcodeVideoWorkflow', () => {
await transcodeVideoWorkflow(task)
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith({
assetKey: 'audio.wav',
filePath: '/tmp/video.mp4',
threads: 4,
})
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith(
expect.objectContaining({
assetKey: 'audio.wav',
filePath: '/tmp/video.mp4',
threads: 4,
}),
)
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-audio',
@@ -390,6 +392,7 @@ describe('transcodeVideoWorkflow', () => {
posterSpec: {
key: 'files/asset-poster/poster.webp',
},
taskId: 'task-poster',
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
@@ -68,6 +68,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
const posterResult = await executeActivity(workerQueue, extractPosterActivity, {
assetKey: key,
posterSpec,
taskId: task.id,
})
mediaInfo.poster = posterResult.poster
await executeActivity(workerQueue, updateAssetMediaActivity, {
@@ -98,6 +99,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
}
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
taskId: task.id,
assetKey: key,
filePath,
videoSpec,
@@ -119,6 +121,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
const isAudio = mediaInfo.proxyType === 'audio'
if (isAudio && metadata) {
const audioTranscode = await executeActivity(workerQueue, transcodeAudioActivity, {
taskId: task.id,
assetKey: key,
filePath,
threads: spec.threads,
@@ -150,6 +153,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
}
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
taskId: task.id,
assetKey: key,
filePath,
spriteSpec,
@@ -48,6 +48,27 @@ describe('Task Activities', () => {
expect(updated?.status).toBe(AssetStatus.processing)
})
it('should NOT overwrite trashed status when asset is soft-deleted (isDeleted: true)', async () => {
const asset = await prisma.asset.create({
data: {
name: 'trashed-video.mp4',
storageKey: { create: { key: 'trashed-video.mp4' } },
status: AssetStatus.trashed,
isDeleted: true,
type: 'file',
},
})
await updateAssetStatusActivity({
assetId: asset.id,
status: AssetStatus.processed,
})
const updated = await prisma.asset.findUnique({ where: { id: asset.id } })
expect(updated?.status).toBe(AssetStatus.trashed)
expect(updated?.isDeleted).toBe(true)
})
it('should update task usage', async () => {
const task = await prisma.workflowTask.create({
data: {
@@ -69,4 +90,13 @@ describe('Task Activities', () => {
expect(updated?.outputTokens).toBe(200)
expect(updated?.model).toBe('gpt-4o')
})
it('should not throw error when updating status of a non-existent or purged task', async () => {
await expect(
updateTaskStatusActivity({
taskId: 'non-existent-task-id',
status: WorkflowTaskStatus.failed,
}),
).resolves.not.toThrow()
})
})
@@ -70,7 +70,7 @@ export interface UpdateTaskStatusParams {
}
export async function updateTaskStatusActivity(params: UpdateTaskStatusParams): Promise<void> {
await prisma.workflowTask.update({
await prisma.workflowTask.updateMany({
where: { id: params.taskId },
data: {
status: params.status,
@@ -85,8 +85,8 @@ export interface UpdateAssetStatusParams {
}
export async function updateAssetStatusActivity(params: UpdateAssetStatusParams): Promise<void> {
await prisma.asset.update({
where: { id: params.assetId },
await prisma.asset.updateMany({
where: { id: params.assetId, isDeleted: false },
data: { status: params.status },
})
}
@@ -99,7 +99,7 @@ export interface UpdateTaskUsageParams {
}
export async function updateTaskUsageActivity(params: UpdateTaskUsageParams): Promise<void> {
await prisma.workflowTask.update({
await prisma.workflowTask.updateMany({
where: { id: params.taskId },
data: {
inputTokens: params.inputTokens,
@@ -120,7 +120,7 @@ export interface UpdateWorkflowTaskParams {
}
export async function updateWorkflowTaskActivity(params: UpdateWorkflowTaskParams): Promise<void> {
await prisma.workflowTask.update({
await prisma.workflowTask.updateMany({
where: { id: params.taskId },
data: {
...(params.status !== undefined ? { status: params.status } : {}),
+1
View File
@@ -4,3 +4,4 @@ export * from './temporal-executor'
export * from './workflow-utils'
export * from './workflow'
export * from './activities/task'
export * from './local-task-storage'
@@ -413,5 +413,153 @@ describe('LocalExecutor Integration Tests', () => {
unregisterLocalCancelHandler(taskId)
})
it('clears active heartbeat timer when cancelling a task', async () => {
let resolveChat: (value: unknown) => void = () => {}
const chatPromise = new Promise((resolve) => {
resolveChat = resolve
})
mocks.agentChat.mockImplementationOnce(() => chatPromise)
const task = await prisma.workflowTask.create({
data: {
assetId: 'test-asset-cancel',
type: WorkflowTaskType.chat,
status: WorkflowTaskStatus.pending,
},
})
// Wait briefly for submit to start and establish heartbeat
await new Promise((resolve) => setTimeout(resolve, 50))
// Heartbeat timer should be tracked
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const heartbeats = (executor as any).heartbeatIntervals as Map<string, Timer>
expect(heartbeats.has(task.id)).toBe(true)
await executor.cancel(task.id)
// Timer should be cleared and removed
expect(heartbeats.has(task.id)).toBe(false)
resolveChat(undefined)
await new Promise((resolve) => setTimeout(resolve, 50))
})
it('handles heartbeat updateMany returning 0 when task is deleted from database', async () => {
let resolveChat: (value: unknown) => void = () => {}
const chatPromise = new Promise((resolve) => {
resolveChat = resolve
})
mocks.agentChat.mockImplementationOnce(() => chatPromise)
const task = await prisma.workflowTask.create({
data: {
assetId: 'test-asset-deleted',
type: WorkflowTaskType.chat,
status: WorkflowTaskStatus.pending,
},
})
await new Promise((resolve) => setTimeout(resolve, 50))
// Delete the task from DB while task is running
await prisma.workflowTask.delete({ where: { id: task.id } })
// UpdateMany should safely return count 0 without throwing P2025
const { count } = await prisma.workflowTask.updateMany({
where: { id: task.id },
data: { heartbeat: new Date() },
})
expect(count).toBe(0)
resolveChat(undefined)
await new Promise((resolve) => setTimeout(resolve, 50))
})
it('aborts local task abort signal when task is cancelled', async () => {
const { getLocalTaskAbortSignal } = await import('./workflow-utils')
let resolveChat: (value: unknown) => void = () => {}
const chatPromise = new Promise((resolve) => {
resolveChat = resolve
})
mocks.agentChat.mockImplementationOnce(() => chatPromise)
const task = await prisma.workflowTask.create({
data: {
assetId: 'test-asset-abort-signal',
type: WorkflowTaskType.chat,
status: WorkflowTaskStatus.pending,
},
})
await new Promise((resolve) => setTimeout(resolve, 50))
const signal = getLocalTaskAbortSignal(task.id)
expect(signal).toBeDefined()
expect(signal?.aborted).toBe(false)
await executor.cancel(task.id)
expect(signal?.aborted).toBe(true)
resolveChat(undefined)
await new Promise((resolve) => setTimeout(resolve, 50))
})
it('cancels queued local tasks before execution when limiter is saturated', async () => {
let resolveFirstTranscode: (value: unknown) => void = () => {}
const firstTranscodePromise = new Promise((resolve) => {
resolveFirstTranscode = resolve
})
mocks.transcodeMedia.mockImplementationOnce(() => firstTranscodePromise)
mocks.transcodeMedia.mockImplementationOnce(() => Promise.resolve())
// 1. Task 1 starts and occupies the single transcode slot
const task1 = await prisma.workflowTask.create({
data: {
assetId: 'asset-queued-cancel-1',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
},
})
// 2. Task 2 is submitted and queued because concurrency limit is 1
const task2 = await prisma.workflowTask.create({
data: {
assetId: 'asset-queued-cancel-2',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
},
})
await new Promise((resolve) => setTimeout(resolve, 50))
// Task 1 should be running
expect(mocks.transcodeMedia).toHaveBeenCalledWith(expect.objectContaining({ id: task1.id }))
// Task 2 should not have started yet
expect(mocks.transcodeMedia).not.toHaveBeenCalledWith(
expect.objectContaining({ id: task2.id }),
)
// 3. Cancel task 2 while it is still waiting in the queue
await executor.cancel(task2.id)
// 4. Release task 1 so the concurrency slot becomes available
resolveFirstTranscode(undefined)
await new Promise((resolve) => setTimeout(resolve, 50))
// 5. Verify task 2's workflow was NEVER executed
expect(mocks.transcodeMedia).not.toHaveBeenCalledWith(
expect.objectContaining({ id: task2.id }),
)
// 6. Verify task 2 was marked failed / cancelled
const dbTask2 = await prisma.workflowTask.findUnique({
where: { id: task2.id },
})
expect(dbTask2?.status).toBe(WorkflowTaskStatus.failed)
expect((dbTask2?.output as { error?: string })?.error).toContain('cancelled')
})
})
})
+108 -11
View File
@@ -2,8 +2,15 @@ import { prisma } from '@shumai/db'
import { WorkflowTask, WorkflowTaskStatus } from '@shumai/db'
import { Executor } from './executor'
import * as taskActivities from './activities/task'
import {
getConcurrencyLimit,
registerLocalTaskAbortController,
runInLocalTaskContext,
triggerLocalCancel,
unregisterLocalTaskAbortController,
} from './workflow-utils'
import { logger } from '@shumai/core/src/logger'
import { getConcurrencyLimit, triggerLocalCancel } from './workflow-utils'
import './local-task-storage'
type WorkflowFn = (task: WorkflowTask) => Promise<void>
@@ -30,6 +37,7 @@ export function registerActivities(acts: Record<string, unknown>) {
globalObj.__localActivities = activityRegistry
globalObj.__localLogger = logger
// ConcurrencyLimiter manages concurrent execution of promises with a FIFO queue.
export class ConcurrencyLimiter {
private activeCount = 0
private queue: (() => void)[] = []
@@ -72,6 +80,7 @@ export class ConcurrencyLimiter {
export class LocalExecutor implements Executor {
private interval: Timer | null = null
private processingTasks = new Set<string>()
private heartbeatIntervals = new Map<string, Timer>()
private transcodeLimiter: ConcurrencyLimiter
private generalLimiter: ConcurrencyLimiter
@@ -94,7 +103,7 @@ export class LocalExecutor implements Executor {
async submit(task: WorkflowTask): Promise<string> {
if (task.status !== WorkflowTaskStatus.pending) {
await prisma.workflowTask.update({
await prisma.workflowTask.updateMany({
where: { id: task.id },
data: { status: WorkflowTaskStatus.pending },
})
@@ -135,14 +144,26 @@ export class LocalExecutor implements Executor {
// 2. Start heartbeat updater immediately
const heartbeatInterval = setInterval(async () => {
try {
await prisma.workflowTask.update({
const { count } = await prisma.workflowTask.updateMany({
where: { id: task.id },
data: { heartbeat: new Date() },
})
if (count === 0) {
clearInterval(heartbeatInterval)
this.heartbeatIntervals.delete(task.id)
logger.debug(
{ taskId: task.id },
'[LocalExecutor] Stopped heartbeat for deleted or cancelled task',
)
}
} catch (err) {
console.error(`[LocalExecutor] Failed to update heartbeat for task ${task.id}:`, err)
}
}, 5000)
this.heartbeatIntervals.set(task.id, heartbeatInterval)
const abortController = new AbortController()
registerLocalTaskAbortController(task.id, abortController)
const limiter =
task.type && task.type.startsWith('transcode')
@@ -152,10 +173,32 @@ export class LocalExecutor implements Executor {
limiter
.run(async () => {
try {
await this.processTaskWrapper(task)
if (abortController.signal.aborted) {
logger.info(
{ taskId: task.id },
'[LocalExecutor] Queued task was aborted before execution',
)
try {
await prisma.workflowTask.updateMany({
where: { id: task.id, status: WorkflowTaskStatus.processing },
data: {
status: WorkflowTaskStatus.failed,
output: { error: 'Workflow task was cancelled before execution' },
},
})
} catch {
// Ignore if task was already deleted
}
return
}
await runInLocalTaskContext({ taskId: task.id, abortController }, async () => {
await this.processTaskWrapper(task)
})
} finally {
// 3. Clean up the heartbeat updater and processing set when the task finishes
clearInterval(heartbeatInterval)
this.heartbeatIntervals.delete(task.id)
unregisterLocalTaskAbortController(task.id)
this.processingTasks.delete(task.id)
}
})
@@ -168,7 +211,13 @@ export class LocalExecutor implements Executor {
}
async cancel(taskId: string): Promise<void> {
const timer = this.heartbeatIntervals.get(taskId)
if (timer) {
clearInterval(timer)
this.heartbeatIntervals.delete(taskId)
}
triggerLocalCancel(taskId)
logger.info({ taskId }, '[LocalExecutor] Cancelled workflow task')
}
start(): void {
@@ -181,6 +230,13 @@ export class LocalExecutor implements Executor {
clearInterval(this.interval)
this.interval = null
}
for (const timer of this.heartbeatIntervals.values()) {
clearInterval(timer)
}
this.heartbeatIntervals.clear()
for (const taskId of this.processingTasks) {
triggerLocalCancel(taskId)
}
}
// Returns the array of task promises so tests can await them to avoid early rollback.
@@ -188,6 +244,10 @@ export class LocalExecutor implements Executor {
const promises: Promise<void>[] = []
try {
const staleTime = new Date(Date.now() - 30 * 1000)
// Find tasks that are:
// 1. Pending (new tasks)
// 2. Processing but stale (worker crashed, heartbeat > 30s ago)
const tasks = await prisma.workflowTask.findMany({
where: {
OR: [
@@ -198,6 +258,7 @@ export class LocalExecutor implements Executor {
},
],
},
orderBy: { id: 'asc' },
take: 50,
})
@@ -206,8 +267,7 @@ export class LocalExecutor implements Executor {
this.processingTasks.add(task.id)
// 1. Immediately mark task as processing and set initial heartbeat in DB
// to guarantee no other ticks/replicas can double-query it.
// We use updateMany to atomically claim it only if it is still pending or stale.
// atomically, ensuring no two workers pick up the same task.
try {
const affected = await prisma.workflowTask.updateMany({
where: {
@@ -230,23 +290,38 @@ export class LocalExecutor implements Executor {
continue
}
} catch (err) {
console.error(`[LocalExecutor] Failed to mark task ${task.id} as processing:`, err)
console.error(
`[LocalExecutor] Failed to mark task ${task.id} as processing in tick:`,
err,
)
this.processingTasks.delete(task.id)
continue
}
// 2. Start heartbeat updater immediately so it updates heartbeat
// 2. Start heartbeat updater immediately to prevent other workers from considering it stale
// even while the task is queued waiting for limiter capacity.
const heartbeatInterval = setInterval(async () => {
try {
await prisma.workflowTask.update({
const { count } = await prisma.workflowTask.updateMany({
where: { id: task.id },
data: { heartbeat: new Date() },
})
if (count === 0) {
clearInterval(heartbeatInterval)
this.heartbeatIntervals.delete(task.id)
logger.debug(
{ taskId: task.id },
'[LocalExecutor] Stopped heartbeat for deleted or cancelled task',
)
}
} catch (err) {
console.error(`[LocalExecutor] Failed to update heartbeat for task ${task.id}:`, err)
}
}, 5000)
this.heartbeatIntervals.set(task.id, heartbeatInterval)
const abortController = new AbortController()
registerLocalTaskAbortController(task.id, abortController)
const limiter =
task.type && task.type.startsWith('transcode')
@@ -256,10 +331,32 @@ export class LocalExecutor implements Executor {
const promise = limiter
.run(async () => {
try {
await this.processTaskWrapper(task)
if (abortController.signal.aborted) {
logger.info(
{ taskId: task.id },
'[LocalExecutor] Queued task was aborted before execution',
)
try {
await prisma.workflowTask.updateMany({
where: { id: task.id, status: WorkflowTaskStatus.processing },
data: {
status: WorkflowTaskStatus.failed,
output: { error: 'Workflow task was cancelled before execution' },
},
})
} catch {
// Ignore if task was already deleted
}
return
}
await runInLocalTaskContext({ taskId: task.id, abortController }, async () => {
await this.processTaskWrapper(task)
})
} finally {
// 3. Clean up the heartbeat updater and processing set when the task finishes
clearInterval(heartbeatInterval)
this.heartbeatIntervals.delete(task.id)
unregisterLocalTaskAbortController(task.id)
this.processingTasks.delete(task.id)
}
})
@@ -285,7 +382,7 @@ export class LocalExecutor implements Executor {
} catch (err) {
console.error(`Error in workflow ${task.type} for task ${task.id}:`, err)
try {
await prisma.workflowTask.update({
await prisma.workflowTask.updateMany({
where: { id: task.id },
data: {
status: WorkflowTaskStatus.failed,
@@ -0,0 +1,12 @@
import { AsyncLocalStorage } from 'node:async_hooks'
import type { LocalTaskContext } from './workflow-utils'
// Store AsyncLocalStorage on globalThis to share with workflow-utils without
// requiring workflow-utils to import node:async_hooks (which would break Temporal's Webpack workflow bundler).
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const globalObj = globalThis as any
if (!globalObj.__localTaskStorage) {
globalObj.__localTaskStorage = new AsyncLocalStorage<LocalTaskContext>()
}
export const localTaskStorage: AsyncLocalStorage<LocalTaskContext> = globalObj.__localTaskStorage
@@ -1,5 +1,6 @@
import { Connection, Client } from '@temporalio/client'
import { WorkflowTask, WorkflowTaskType } from '@shumai/db'
import { prisma, WorkflowTask, WorkflowTaskType } from '@shumai/db'
import { logger } from '@shumai/core/src/logger'
import { Executor } from './executor'
import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils'
@@ -80,9 +81,18 @@ export class TemporalExecutor implements Executor {
async cancel(taskId: string): Promise<void> {
const client = await this.getClient()
const workflowId = `chat-${taskId}`
const handle = client.workflow.getHandle(workflowId)
await handle.cancel()
const task = await prisma.workflowTask.findUnique({
where: { id: taskId },
select: { type: true },
})
const workflowId = task?.type ? `${task.type}-${taskId}` : `chat-${taskId}`
try {
const handle = client.workflow.getHandle(workflowId)
await handle.cancel()
logger.info({ taskId, workflowId }, '[TemporalExecutor] Cancelled workflow task')
} catch (err) {
logger.warn({ taskId, workflowId, err }, '[TemporalExecutor] Failed to cancel workflow task')
}
}
start(): void {
+60 -2
View File
@@ -204,6 +204,11 @@ export function getConcurrencyLimit(envVar: string | undefined, defaultValue: nu
return parsed
}
export interface LocalTaskContext {
taskId: string
abortController: AbortController
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const globalObjWithCancel = globalThis as any
if (!globalObjWithCancel.__localCancelHandlers) {
@@ -211,6 +216,52 @@ if (!globalObjWithCancel.__localCancelHandlers) {
}
const localCancelHandlers: Map<string, () => void> = globalObjWithCancel.__localCancelHandlers
if (!globalObjWithCancel.__localTaskAbortControllers) {
globalObjWithCancel.__localTaskAbortControllers = new Map<string, AbortController>()
}
const localTaskAbortControllers: Map<string, AbortController> =
globalObjWithCancel.__localTaskAbortControllers
interface LocalTaskStorageHolder {
getStore: () => LocalTaskContext | undefined
run: <T>(store: LocalTaskContext, fn: () => T) => T
}
function getLocalTaskStorage(): LocalTaskStorageHolder | undefined {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return (globalThis as any).__localTaskStorage
}
export function registerLocalTaskAbortController(
taskId: string,
controller: AbortController,
): void {
localTaskAbortControllers.set(taskId, controller)
}
export function unregisterLocalTaskAbortController(taskId: string): void {
localTaskAbortControllers.delete(taskId)
}
export function runInLocalTaskContext<T>(context: LocalTaskContext, fn: () => T): T {
registerLocalTaskAbortController(context.taskId, context.abortController)
const storage = getLocalTaskStorage()
if (storage) {
return storage.run(context, fn)
}
return fn()
}
export function getLocalTaskAbortSignal(taskId?: string): AbortSignal | undefined {
if (taskId) {
const controller = localTaskAbortControllers.get(taskId)
if (controller) {
return controller.signal
}
}
return getLocalTaskStorage()?.getStore()?.abortController?.signal
}
export function registerLocalCancelHandler(taskId: string, cancelFn: () => void) {
localCancelHandlers.set(taskId, cancelFn)
}
@@ -220,10 +271,17 @@ export function unregisterLocalCancelHandler(taskId: string) {
}
export function triggerLocalCancel(taskId: string): boolean {
let cancelled = false
const controller = localTaskAbortControllers.get(taskId)
if (controller) {
controller.abort()
localTaskAbortControllers.delete(taskId)
cancelled = true
}
const handler = localCancelHandlers.get(taskId)
if (handler) {
handler()
return true
cancelled = true
}
return false
return cancelled
}
+4 -1
View File
@@ -1,4 +1,4 @@
import { prisma, registerWorkflowTrigger, WorkflowTask } from '@shumai/db'
import { prisma, registerWorkflowCancel, registerWorkflowTrigger, WorkflowTask } from '@shumai/db'
import path from 'path'
import { ulid } from 'ulid'
import * as taskActivities from './activities/task'
@@ -187,3 +187,6 @@ export const workflowService = new WorkflowService()
registerWorkflowTrigger(async (task) => {
await workflowService.submit(task)
})
registerWorkflowCancel(async (taskId) => {
await workflowService.cancel(taskId)
})