mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 13:58:00 +00:00
fix(transcode): preserve trashed status during background jobs and handle permanent purge (#416)
* fix(transcode): preserve trashed status during background jobs and handle permanent purge * fix(workflow): clear heartbeat and add info logs on task cancellation * fix(transcode): terminate media processes and guard task activities on cancellation - Pass AbortSignal to ffmpeg and pdftoppm child processes in transcodeService to terminate processes on task cancellation - Add local task AbortController context and cancellation guards in executeActivity to prevent subsequent workflow activities - Forward taskId and cancellation signals in transcode, screenshot, and PDF activities - Use updateMany instead of update in task activities to eliminate Prisma P2025 errors on purged task records * fix(workflow): allow cleanup activities in finally blocks during task cancellation Remove blanket cancellation check in executeActivity so that workflows can execute cleanup and failure handling activities in catch and finally blocks upon cancellation, while relying on AbortSignal in active activities to abort execution. * docs(changelog): improve user friendliness of changelog entries and formatting example * docs(agents): update changelog guidelines to emphasize end-user friendly descriptions * fix(transcode): handle queued cancellation and delete races in media workflow
This commit is contained in:
@@ -68,8 +68,8 @@ We maintain `CHANGELOG.md` following [Keep a Changelog](https://keepachangelog.c
|
||||
|
||||
Before opening a pull request, add an entry under `## [Unreleased]` for any user-facing changes:
|
||||
|
||||
- **Focus on User Impact**: Describe the user-facing outcome or observable behavior in plain terms, not internal code mechanics, file paths, or private symbols.
|
||||
- **Formatting**: `- **<scope>**: <Description>` (e.g. `- **comment**: Add emoji reaction support for comments`).
|
||||
- **End-User Friendly**: Make entries easy to understand for end users. For new features, explain what they actually do in plain language without heavy technical jargon. For bug fixes, describe what the bug was in a friendly, relatable way rather than focusing on internal code mechanics, file paths, or private symbols.
|
||||
- **Formatting**: `- **<scope>**: <Description>` (e.g. `- **asset**: Fix an issue where moving a video to trash while it was still transcoding could cause it to lose its trashed status`).
|
||||
- **No PR Numbers**: Do not include PR numbers or links; the release pipeline automatically appends the detailed PR list.
|
||||
- **Internal Changes**: Routine refactors, test additions, or internal chores without external behavior changes may omit changelog updates.
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **asset**: Fix an issue where moving a video to trash while it was still transcoding could cause it to lose its trashed status
|
||||
- **transcode**: Immediately stop background video processing and transcoding when a file is permanently deleted from the trash
|
||||
- **asset**: Fix proxy download filename using original file extension instead of proxy extension
|
||||
- **webui**: Center breadcrumb file action popup menu on the chevron icon
|
||||
|
||||
|
||||
@@ -3,13 +3,14 @@ import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
|
||||
import { AssetType, AssetStatus, Prisma } from '@shumai/db'
|
||||
import { AssetType, AssetStatus, Prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { uploadService } from '@shumai/core/src/upload/upload'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { AssetService } from './asset'
|
||||
|
||||
import { ensureJpegInStorage } from '@shumai/core/src/s3/preview-converter'
|
||||
import { workflowService } from '@shumai/workflow-core'
|
||||
|
||||
vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
s3Service: {
|
||||
@@ -3004,6 +3005,65 @@ describe('AssetService — natural sort by name', () => {
|
||||
|
||||
await expect(assetService.emptyTrash(project.id)).resolves.not.toThrow()
|
||||
})
|
||||
|
||||
it('should purge trashed assets whose status was processed, cancel their in-flight tasks, and delete WorkflowTasks', async () => {
|
||||
const cancelSpy = vi.spyOn(workflowService, 'cancel').mockResolvedValue(undefined)
|
||||
|
||||
const team = await prisma.team.create({ data: { name: 'Purge Task Team' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Purge Task Project', teamId: team.id },
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Test User Purge', email: `test-purge-${Date.now()}@example.com` },
|
||||
})
|
||||
|
||||
// Asset was deleted, but status was set to 'processed' by background activity
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'skewed-status.mp4',
|
||||
type: AssetType.file,
|
||||
status: AssetStatus.processed,
|
||||
isDeleted: true,
|
||||
deletedAt: new Date(),
|
||||
project: { connect: { id: project.id } },
|
||||
parent: { connect: { id: rootFolder.id } },
|
||||
creator: { connect: { id: user.id } },
|
||||
storageKey: { create: { key: 'files/skewed-status/raw.mp4' } },
|
||||
},
|
||||
})
|
||||
|
||||
// In-flight task for this asset
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: file.id,
|
||||
type: WorkflowTaskType.transcode_video,
|
||||
status: WorkflowTaskStatus.processing,
|
||||
},
|
||||
})
|
||||
|
||||
await assetService.emptyTrash(project.id)
|
||||
|
||||
// Asset should be deleted from DB
|
||||
const assetInDb = await prisma.asset.findUnique({ where: { id: file.id } })
|
||||
expect(assetInDb).toBeNull()
|
||||
|
||||
// Task should be cancelled
|
||||
expect(cancelSpy).toHaveBeenCalledWith(task.id)
|
||||
|
||||
// WorkflowTask should be deleted from DB
|
||||
const taskInDb = await prisma.workflowTask.findUnique({ where: { id: task.id } })
|
||||
expect(taskInDb).toBeNull()
|
||||
|
||||
cancelSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
describe('expireTrashedAssets', () => {
|
||||
@@ -4904,4 +4964,88 @@ describe('AssetService — natural sort by name', () => {
|
||||
expect(stackInfo?.commentsCount).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('restoreAssets with in-flight transcoding', () => {
|
||||
it('should restore asset with status "processing" if a transcode task is still active', async () => {
|
||||
const team = await prisma.team.create({ data: { name: 'Restore Transcode Team' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Restore Transcode Project', teamId: team.id },
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'transcoding-video.mp4',
|
||||
type: AssetType.file,
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
status: AssetStatus.trashed,
|
||||
isDeleted: true,
|
||||
deletedAt: new Date(),
|
||||
},
|
||||
})
|
||||
|
||||
// Create an active transcode task
|
||||
await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: file.id,
|
||||
type: WorkflowTaskType.transcode_video,
|
||||
status: WorkflowTaskStatus.processing,
|
||||
},
|
||||
})
|
||||
|
||||
await assetService.restoreAssets([file.id])
|
||||
|
||||
const restored = await prisma.asset.findUnique({ where: { id: file.id } })
|
||||
expect(restored?.isDeleted).toBe(false)
|
||||
expect(restored?.status).toBe(AssetStatus.processing)
|
||||
})
|
||||
|
||||
it('should restore asset with status "processed" if no active transcode task exists', async () => {
|
||||
const team = await prisma.team.create({ data: { name: 'Restore Completed Team' } })
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Restore Completed Project', teamId: team.id },
|
||||
})
|
||||
const rootFolder = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'root',
|
||||
type: AssetType.folder,
|
||||
projectId: project.id,
|
||||
status: AssetStatus.uploaded,
|
||||
},
|
||||
})
|
||||
const file = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'completed-video.mp4',
|
||||
type: AssetType.file,
|
||||
projectId: project.id,
|
||||
parentId: rootFolder.id,
|
||||
status: AssetStatus.trashed,
|
||||
isDeleted: true,
|
||||
deletedAt: new Date(),
|
||||
},
|
||||
})
|
||||
|
||||
// Completed transcode task
|
||||
await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: file.id,
|
||||
type: WorkflowTaskType.transcode_video,
|
||||
status: WorkflowTaskStatus.completed,
|
||||
},
|
||||
})
|
||||
|
||||
await assetService.restoreAssets([file.id])
|
||||
|
||||
const restored = await prisma.asset.findUnique({ where: { id: file.id } })
|
||||
expect(restored?.isDeleted).toBe(false)
|
||||
expect(restored?.status).toBe(AssetStatus.processed)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { prisma } from '@shumai/db'
|
||||
import { cancelWorkflowTask, prisma } from '@shumai/db'
|
||||
import {
|
||||
AncestorFolder,
|
||||
AssetInfo,
|
||||
@@ -28,7 +28,15 @@ import {
|
||||
type ExportOptions,
|
||||
type ExportResult,
|
||||
} from './comments-export'
|
||||
import { type Asset, AssetStatus, AssetType, Prisma, type StorageKey } from '@shumai/db'
|
||||
import {
|
||||
type Asset,
|
||||
AssetStatus,
|
||||
AssetType,
|
||||
Prisma,
|
||||
type StorageKey,
|
||||
WorkflowTaskStatus,
|
||||
WorkflowTaskType,
|
||||
} from '@shumai/db'
|
||||
import { HTTPException } from 'hono/http-exception'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { PaginatedData, paginateQuery, PaginationParams } from '@shumai/core/src/pagination'
|
||||
@@ -1343,7 +1351,7 @@ export class AssetService {
|
||||
private async cascadeStatusToPendingPurge(rootIds: string[]): Promise<void> {
|
||||
if (rootIds.length === 0) return
|
||||
|
||||
await this.prismaClient.$executeRaw`
|
||||
const rows = await this.prismaClient.$queryRaw<{ id: string }[]>`
|
||||
WITH RECURSIVE descendant AS (
|
||||
SELECT id FROM assets WHERE id = ANY(${rootIds})
|
||||
UNION ALL
|
||||
@@ -1358,15 +1366,35 @@ export class AssetService {
|
||||
WHERE s.type = 'symlink' AND s.target_id IN (SELECT id FROM all_purging_targets)
|
||||
)
|
||||
UPDATE assets SET status = 'pending_purge', updated_at = NOW()
|
||||
WHERE id IN (SELECT id FROM all_purging_targets) OR id IN (SELECT id FROM all_affected_symlinks);
|
||||
WHERE id IN (SELECT id FROM all_purging_targets) OR id IN (SELECT id FROM all_affected_symlinks)
|
||||
RETURNING id;
|
||||
`
|
||||
|
||||
const affectedIds = rows.map((r) => r.id)
|
||||
if (affectedIds.length > 0) {
|
||||
const inFlightTasks = await this.prismaClient.workflowTask.findMany({
|
||||
where: {
|
||||
assetId: { in: affectedIds },
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
for (const task of inFlightTasks) {
|
||||
try {
|
||||
logger.info({ taskId: task.id }, 'Cancelling in-flight workflow task for purging asset')
|
||||
await cancelWorkflowTask(task.id)
|
||||
} catch (err) {
|
||||
logger.warn({ taskId: task.id, err }, 'Failed to cancel workflow task during purge')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async emptyTrash(projectId: string): Promise<void> {
|
||||
const trashedRoots = await this.prismaClient.asset.findMany({
|
||||
where: {
|
||||
projectId,
|
||||
status: 'trashed',
|
||||
isDeleted: true,
|
||||
},
|
||||
select: { id: true },
|
||||
@@ -1418,9 +1446,9 @@ export class AssetService {
|
||||
// Find up to 100 expired roots, ordered by oldest first to avoid starvation
|
||||
const expiredRoots = await this.prismaClient.asset.findMany({
|
||||
where: {
|
||||
status: 'trashed',
|
||||
deletedAt: { lt: thirtyDaysAgo },
|
||||
isDeleted: true,
|
||||
status: { not: AssetStatus.pending_purge },
|
||||
deletedAt: { lt: thirtyDaysAgo },
|
||||
},
|
||||
orderBy: { deletedAt: 'asc' },
|
||||
select: { id: true },
|
||||
@@ -1435,6 +1463,35 @@ export class AssetService {
|
||||
* Stage 2: Delete DB records for assets marked as 'pending_purge'.
|
||||
*/
|
||||
private async purgePendingAssets() {
|
||||
const pendingAssets = await this.prismaClient.asset.findMany({
|
||||
where: { status: AssetStatus.pending_purge },
|
||||
select: { id: true },
|
||||
})
|
||||
const pendingAssetIds = pendingAssets.map((a) => a.id)
|
||||
|
||||
if (pendingAssetIds.length > 0) {
|
||||
const inFlightTasks = await this.prismaClient.workflowTask.findMany({
|
||||
where: {
|
||||
assetId: { in: pendingAssetIds },
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
|
||||
for (const task of inFlightTasks) {
|
||||
try {
|
||||
logger.info({ taskId: task.id }, 'Cancelling in-flight workflow task before purge')
|
||||
await cancelWorkflowTask(task.id)
|
||||
} catch (err) {
|
||||
logger.warn({ taskId: task.id, err }, 'Failed to cancel workflow task during purge')
|
||||
}
|
||||
}
|
||||
|
||||
await this.prismaClient.workflowTask.deleteMany({
|
||||
where: { assetId: { in: pendingAssetIds } },
|
||||
})
|
||||
}
|
||||
|
||||
const { count } = await this.prismaClient.asset.deleteMany({
|
||||
where: { status: AssetStatus.pending_purge },
|
||||
})
|
||||
@@ -1577,11 +1634,30 @@ export class AssetService {
|
||||
}
|
||||
}
|
||||
|
||||
const activeTask = await tx.workflowTask.findFirst({
|
||||
where: {
|
||||
assetId: a.id,
|
||||
status: { in: [WorkflowTaskStatus.pending, WorkflowTaskStatus.processing] },
|
||||
type: {
|
||||
in: [
|
||||
WorkflowTaskType.transcode,
|
||||
WorkflowTaskType.transcode_video,
|
||||
WorkflowTaskType.transcode_image,
|
||||
WorkflowTaskType.transcode_pdf,
|
||||
WorkflowTaskType.transcode_pdf_pages,
|
||||
WorkflowTaskType.transcode_screenshot,
|
||||
WorkflowTaskType.transcode_image_annotation,
|
||||
WorkflowTaskType.transcode_watermark,
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await tx.asset.update({
|
||||
where: { id: a.id },
|
||||
data: {
|
||||
isDeleted: false,
|
||||
status: 'processed',
|
||||
status: activeTask ? AssetStatus.processing : AssetStatus.processed,
|
||||
deletedAt: null,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1776,5 +1776,41 @@ describe('TranscodeService', () => {
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
|
||||
it('transcodeVideo with signal should pass signal to execFile', async () => {
|
||||
vi.mocked(execFile).mockImplementation(
|
||||
(
|
||||
_cmd: unknown,
|
||||
_args: unknown,
|
||||
optionsOrCallback: unknown,
|
||||
maybeCallback?: unknown,
|
||||
): ReturnType<typeof child_process.execFile> => {
|
||||
const cb = (
|
||||
typeof optionsOrCallback === 'function' ? optionsOrCallback : maybeCallback
|
||||
) as (err: Error | null, result: { stdout: string; stderr: string }) => void
|
||||
if (typeof cb === 'function') {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
return {} as ReturnType<typeof child_process.execFile>
|
||||
},
|
||||
)
|
||||
|
||||
const controller = new AbortController()
|
||||
const outputFile = path.join(tempDir, 'out_signal.mp4')
|
||||
await transcodeService.transcodeVideo({
|
||||
inputFile: 'input.mp4',
|
||||
outputFile,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
signal: controller.signal,
|
||||
})
|
||||
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'ffmpeg',
|
||||
expect.any(Array),
|
||||
{ signal: controller.signal },
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -117,6 +117,7 @@ export interface TranscodeVideoParams {
|
||||
videoBitrate?: string
|
||||
sourceVideoBitrate?: number
|
||||
threads?: number
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export interface EncoderConfig {
|
||||
@@ -634,7 +635,13 @@ export class TranscodeService {
|
||||
|
||||
args.push('-movflags', '+faststart', '-max_muxing_queue_size', '1024', params.outputFile)
|
||||
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
if (params.signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
|
||||
signal: params.signal,
|
||||
})
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
}
|
||||
|
||||
async transcodeImage(
|
||||
@@ -741,6 +748,7 @@ export class TranscodeService {
|
||||
outputSprite: string,
|
||||
outputPoster: string,
|
||||
duration: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const spriteFps = 100 / duration
|
||||
const filterComplex = `[0:v]fps=${spriteFps},scale=w=300:h=-2,tile=10x10[sprite_out];[0:v]scale=-2:300:force_original_aspect_ratio=decrease,select='eq(n\\,0)'[thumb_out]`
|
||||
@@ -771,19 +779,30 @@ export class TranscodeService {
|
||||
'1024',
|
||||
outputPoster,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
if (signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
}
|
||||
|
||||
async generatePdfSprite(
|
||||
inputFile: string,
|
||||
outputSprite: string,
|
||||
outputPoster: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ pageCount: number; originalWidth: number; originalHeight: number }> {
|
||||
const tmpDir = this.createTempDir('pdf-sprite-')
|
||||
try {
|
||||
const pagePrefix = path.join(tmpDir, 'page')
|
||||
try {
|
||||
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix])
|
||||
if (signal) {
|
||||
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix], {
|
||||
signal,
|
||||
})
|
||||
} else {
|
||||
await execFileAsync('pdftoppm', ['-png', '-f', '1', '-l', '100', inputFile, pagePrefix])
|
||||
}
|
||||
} catch (err) {
|
||||
const errCode = (err as Record<string, unknown>)?.code
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
@@ -840,7 +859,11 @@ export class TranscodeService {
|
||||
'75',
|
||||
outputSprite,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs])
|
||||
if (signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs], { signal })
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...spriteArgs])
|
||||
}
|
||||
|
||||
await sharp(firstPagePath, { limitInputPixels: false })
|
||||
.toColorspace('srgb')
|
||||
@@ -1090,6 +1113,7 @@ export class TranscodeService {
|
||||
outputFile: string
|
||||
bitrate?: string
|
||||
threads?: number
|
||||
signal?: AbortSignal
|
||||
}): Promise<void> {
|
||||
const bitrate = params.bitrate || '128k'
|
||||
const args = ['-i', params.inputFile, '-vn', '-c:a', 'aac', '-b:a', bitrate, '-ac', '2']
|
||||
@@ -1097,7 +1121,13 @@ export class TranscodeService {
|
||||
args.push('-threads', params.threads.toString())
|
||||
}
|
||||
args.push(params.outputFile)
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
if (params.signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
|
||||
signal: params.signal,
|
||||
})
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
}
|
||||
|
||||
async extractVideoFrames(params: ExtractVideoFramesParams): Promise<string[]> {
|
||||
@@ -1198,6 +1228,7 @@ export class TranscodeService {
|
||||
count: number
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
signal?: AbortSignal
|
||||
}): Promise<Array<{ key: string; timestamp: number }>> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const tmpDir = this.createTempDir('screenshot-')
|
||||
@@ -1259,7 +1290,13 @@ export class TranscodeService {
|
||||
'80',
|
||||
localShotPath,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
if (params.signal) {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], {
|
||||
signal: params.signal,
|
||||
})
|
||||
} else {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
|
||||
const isMatch =
|
||||
commentTimestamp !== undefined &&
|
||||
@@ -1295,6 +1332,7 @@ export class TranscodeService {
|
||||
end: number
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
signal?: AbortSignal
|
||||
}): Promise<Array<{ key: string; page: number }>> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const tmpDir = this.createTempDir('pdf-pages-')
|
||||
@@ -1307,7 +1345,7 @@ export class TranscodeService {
|
||||
// 2. Render pages via pdftoppm (-png -f start -l end)
|
||||
const pagePrefix = path.join(tmpDir, 'page')
|
||||
try {
|
||||
await execFileAsync('pdftoppm', [
|
||||
const pdftoppmArgs = [
|
||||
'-png',
|
||||
'-f',
|
||||
params.start.toString(),
|
||||
@@ -1315,7 +1353,12 @@ export class TranscodeService {
|
||||
params.end.toString(),
|
||||
pdfPath,
|
||||
pagePrefix,
|
||||
])
|
||||
]
|
||||
if (params.signal) {
|
||||
await execFileAsync('pdftoppm', pdftoppmArgs, { signal: params.signal })
|
||||
} else {
|
||||
await execFileAsync('pdftoppm', pdftoppmArgs)
|
||||
}
|
||||
} catch (err) {
|
||||
const errCode = (err as Record<string, unknown>)?.code
|
||||
const msg = err instanceof Error ? err.message : String(err)
|
||||
|
||||
@@ -18,6 +18,20 @@ export function registerWorkflowTrigger(cb: WorkflowTriggerCallback) {
|
||||
onWorkflowTaskCreated = cb
|
||||
}
|
||||
|
||||
// Global callback for workflow task cancellation (decouples db/core packages from workflow engine)
|
||||
type WorkflowCancelCallback = (taskId: string) => Promise<void>
|
||||
let onWorkflowTaskCancelled: WorkflowCancelCallback | null = null
|
||||
|
||||
export function registerWorkflowCancel(cb: WorkflowCancelCallback) {
|
||||
onWorkflowTaskCancelled = cb
|
||||
}
|
||||
|
||||
export async function cancelWorkflowTask(taskId: string): Promise<void> {
|
||||
if (onWorkflowTaskCancelled) {
|
||||
await onWorkflowTaskCancelled(taskId)
|
||||
}
|
||||
}
|
||||
|
||||
const isTest = process.env.NODE_ENV === 'test'
|
||||
|
||||
const globalForPrisma = globalThis as unknown as {
|
||||
|
||||
@@ -595,6 +595,83 @@ describe('Transcode Activities', () => {
|
||||
expect(updated?.hasJpegPreview).toBe(false)
|
||||
expect(s3Service.putObject).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge in updateAssetMediaActivity', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'purging.mp4',
|
||||
storageKey: { create: { key: 'purging.mp4' } },
|
||||
status: 'pending_purge',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
|
||||
await expect(
|
||||
updateAssetMediaActivity({
|
||||
assetId: asset.id,
|
||||
mediaInfo: { duration: 100 } as unknown as PrismaJson.MediaInfo,
|
||||
}),
|
||||
).rejects.toThrow('Asset not found, has no key, or is being purged')
|
||||
|
||||
expect(putObjectSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should tolerate missing or deleted asset without throwing P2025 in updateAssetMediaActivity', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'deleted-during-update.mp4',
|
||||
storageKey: { create: { key: 'deleted-during-update.mp4' } },
|
||||
status: 'uploaded',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
putObjectSpy.mockImplementationOnce(async (): Promise<any> => {
|
||||
await prisma.asset.delete({ where: { id: asset.id } })
|
||||
return {}
|
||||
})
|
||||
|
||||
await expect(
|
||||
updateAssetMediaActivity({
|
||||
assetId: asset.id,
|
||||
mediaInfo: { duration: 100 } as unknown as PrismaJson.MediaInfo,
|
||||
}),
|
||||
).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge in transcodeVideoActivity', async () => {
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'purging-video.mp4',
|
||||
storageKey: { create: { key: 'purging-video.mp4' } },
|
||||
status: 'pending_purge',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
|
||||
await expect(
|
||||
transcodeVideoActivity({
|
||||
assetKey: 'purging-video.mp4',
|
||||
filePath: '/tmp/purging-video.mp4',
|
||||
videoSpec: { resolution: '720p', width: 1280, height: 720 },
|
||||
duration: 10,
|
||||
originalFps: 30,
|
||||
}),
|
||||
).rejects.toThrow('Asset or storage key has been purged or is pending purge')
|
||||
|
||||
expect(putObjectSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should call transcodeService.transcodeVideo', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
|
||||
@@ -733,6 +810,26 @@ describe('Transcode Activities', () => {
|
||||
).rejects.toThrowError(/Video transcoding failed/)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when transcodeVideoActivity is cancelled', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
const abortError = new Error('The operation was aborted')
|
||||
abortError.name = 'AbortError'
|
||||
vi.mocked(transcodeService.transcodeVideo).mockRejectedValue(abortError)
|
||||
|
||||
const promise = transcodeVideoActivity({
|
||||
assetKey: 'v.mp4',
|
||||
filePath: '/tmp/v.mp4',
|
||||
videoSpec: { resolution: '720p', width: 1280, height: 720 },
|
||||
duration: 10,
|
||||
originalFps: 30,
|
||||
})
|
||||
|
||||
await expect(promise).rejects.toThrowError(/Video transcoding cancelled/)
|
||||
await expect(promise).rejects.toMatchObject({
|
||||
nonRetryable: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when transcodeImageActivity fails with sharp error', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(transcodeService.transcodeImage).mockRejectedValue(new Error('sharp failed'))
|
||||
@@ -960,16 +1057,15 @@ describe('Transcode Activities', () => {
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(
|
||||
file: string,
|
||||
args: string[],
|
||||
cb: (err: Error | null, result: { stdout: string; stderr: string }) => void,
|
||||
) => {
|
||||
(file: string, args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
|
||||
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
|
||||
if (file === 'ffmpeg') {
|
||||
const outPath = args[args.length - 1]
|
||||
fs.writeFileSync(outPath, 'fake-poster-bytes')
|
||||
}
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
if (typeof cb === 'function') {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
@@ -992,6 +1088,7 @@ describe('Transcode Activities', () => {
|
||||
'-c:v',
|
||||
'libwebp',
|
||||
]),
|
||||
expect.objectContaining({ signal: undefined }),
|
||||
expect.any(Function),
|
||||
)
|
||||
expect(s3Service.putObject).toHaveBeenCalledWith(
|
||||
@@ -1003,6 +1100,96 @@ describe('Transcode Activities', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when poster extraction is cancelled with aborted signal', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
await expect(
|
||||
extractPosterActivity({
|
||||
assetKey: 'video.mp4',
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure when poster extraction ffmpeg execution is aborted', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
|
||||
const controller = new AbortController()
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(_file: string, _args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
|
||||
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
|
||||
controller.abort()
|
||||
const abortErr = new Error('The operation was aborted')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(abortErr as any).code = 'ABORT_ERR'
|
||||
if (typeof cb === 'function') {
|
||||
cb(abortErr, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await expect(
|
||||
extractPosterActivity({
|
||||
assetKey: 'video.mp4',
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('should throw non-retryable ApplicationFailure and not upload when asset is pending_purge before poster upload', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(s3Service.resolveInput).mockResolvedValue('https://mock-r2.com/video.mp4')
|
||||
const putObjectSpy = vi.mocked(s3Service.putObject)
|
||||
putObjectSpy.mockClear()
|
||||
|
||||
await prisma.asset.create({
|
||||
data: {
|
||||
name: 'purging-poster.mp4',
|
||||
storageKey: { create: { key: 'purging-poster.mp4' } },
|
||||
status: 'pending_purge',
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation(
|
||||
(file: string, args: string[], optionsOrCb: unknown, maybeCb?: unknown) => {
|
||||
const cb = typeof optionsOrCb === 'function' ? optionsOrCb : maybeCb
|
||||
if (file === 'ffmpeg') {
|
||||
const outPath = args[args.length - 1]
|
||||
fs.writeFileSync(outPath, 'fake-poster-bytes')
|
||||
}
|
||||
if (typeof cb === 'function') {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await expect(
|
||||
extractPosterActivity({
|
||||
assetKey: 'purging-poster.mp4',
|
||||
posterSpec: { key: 'files/asset-1/poster.webp' },
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: 'Asset or storage key has been purged or is pending purge',
|
||||
nonRetryable: true,
|
||||
})
|
||||
|
||||
expect(putObjectSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should delete S3 object successfully', async () => {
|
||||
await deleteS3ObjectActivity({ key: 'files/test-key.mp4' })
|
||||
expect(s3Service.deleteObject).toHaveBeenCalledWith('shumai', 'files/test-key.mp4')
|
||||
@@ -1251,5 +1438,56 @@ describe('Transcode Activities', () => {
|
||||
})
|
||||
expect(task).toBeNull()
|
||||
})
|
||||
|
||||
it('should create ai_metadata_autofill task when asset is in trash (isDeleted: true)', async () => {
|
||||
const user = await prisma.user.create({
|
||||
data: { name: 'Test User Trashed Autofill', email: 'test-trashed-autofill@test.com' },
|
||||
})
|
||||
const team = await prisma.team.create({
|
||||
data: { name: 'Test Team Trashed Autofill' },
|
||||
})
|
||||
await prisma.teamMember.create({
|
||||
data: { teamId: team.id, userId: user.id, role: 'owner' },
|
||||
})
|
||||
const project = await prisma.project.create({
|
||||
data: { name: 'Test Project Trashed Autofill', teamId: team.id },
|
||||
})
|
||||
const autofillAgent = await prisma.agent.create({
|
||||
data: {
|
||||
id: user.id,
|
||||
type: 'autofill',
|
||||
enabled: true,
|
||||
teamId: team.id,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
config: { provider: 'test', model: 'test' } as any,
|
||||
},
|
||||
})
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'trashed-video.mp4',
|
||||
mediaType: 'video/mp4',
|
||||
type: 'file',
|
||||
status: 'trashed',
|
||||
isDeleted: true,
|
||||
projectId: project.id,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
media: { proxyType: 'video' } as any,
|
||||
},
|
||||
})
|
||||
|
||||
await createAutofillTaskIfEnabledActivity({
|
||||
assetId: asset.id,
|
||||
teamId: team.id,
|
||||
projectId: project.id,
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.findFirst({
|
||||
where: { assetId: asset.id, type: 'ai_metadata_autofill' },
|
||||
})
|
||||
expect(task).toBeDefined()
|
||||
expect(task?.status).toBe('pending')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((task?.payload as any)?.agent?.agentId).toBe(autofillAgent.id)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { AssetStatus, prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
@@ -12,7 +12,17 @@ import {
|
||||
isMarkdownDocument,
|
||||
isOfficeDocument,
|
||||
} from '@shumai/core/src/utils/mime'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { ApplicationFailure, Context } from '@temporalio/activity'
|
||||
import { getLocalTaskAbortSignal } from '@shumai/workflow-core'
|
||||
|
||||
function getActivityCancellationSignal(taskId?: string): AbortSignal | undefined {
|
||||
try {
|
||||
return Context.current().cancellationSignal
|
||||
} catch {
|
||||
return getLocalTaskAbortSignal(taskId)
|
||||
}
|
||||
}
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
@@ -47,6 +57,24 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureAssetNotPurging(assetKey: string): Promise<void> {
|
||||
const sk = await prisma.storageKey.findUnique({
|
||||
where: { key: assetKey },
|
||||
include: { assets: { select: { id: true, status: true } } },
|
||||
})
|
||||
if (
|
||||
sk &&
|
||||
(sk.status === 'purging' ||
|
||||
sk.assets.length === 0 ||
|
||||
sk.assets.every((a) => a.status === AssetStatus.pending_purge))
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset or storage key has been purged or is pending purge',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export async function getMediaInfoActivity(params: {
|
||||
filePath: string
|
||||
assetId: string
|
||||
@@ -216,6 +244,7 @@ export async function getMediaInfoActivity(params: {
|
||||
}
|
||||
|
||||
export interface VideoActivityParams {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
videoSpec: PrismaJson.VideoTranscode
|
||||
@@ -234,6 +263,14 @@ export async function transcodeVideoActivity(
|
||||
const res = params.videoSpec.resolution || `${params.videoSpec.height}p`
|
||||
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-${res}.mp4`)
|
||||
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Video transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, key)
|
||||
return { ...params.videoSpec, key }
|
||||
@@ -269,10 +306,12 @@ export async function transcodeVideoActivity(
|
||||
hardwareAcceleration: params.hardwareAcceleration,
|
||||
sourceVideoBitrate: params.sourceVideoBitrate,
|
||||
threads: params.threads,
|
||||
signal,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outputFile)
|
||||
const stream = Bun.file(outputFile).stream()
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
|
||||
|
||||
return { ...params.videoSpec, key }
|
||||
@@ -280,6 +319,19 @@ export async function transcodeVideoActivity(
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Video transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -302,6 +354,7 @@ export async function transcodeVideoActivity(
|
||||
}
|
||||
|
||||
export interface AudioActivityParams {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
threads?: number
|
||||
@@ -314,6 +367,14 @@ export async function transcodeAudioActivity(
|
||||
const stem = stemFromKey(params.assetKey)
|
||||
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
|
||||
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Audio transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, key)
|
||||
return { width: 0, height: 0, key }
|
||||
@@ -330,10 +391,12 @@ export async function transcodeAudioActivity(
|
||||
outputFile,
|
||||
bitrate: '128k',
|
||||
threads: params.threads,
|
||||
signal,
|
||||
})
|
||||
|
||||
const stat = fs.statSync(outputFile)
|
||||
const stream = Bun.file(outputFile).stream()
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, key, stream, stat.size, 'video/mp4')
|
||||
|
||||
return { width: 0, height: 0, key }
|
||||
@@ -341,6 +404,19 @@ export async function transcodeAudioActivity(
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Audio transcoding cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -400,6 +476,7 @@ export async function transcodeImageActivity(
|
||||
},
|
||||
)
|
||||
const buffer = fs.readFileSync(outputFile)
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, key, buffer, buffer.length, 'image/webp')
|
||||
|
||||
return { ...params.imageSpec, key }
|
||||
@@ -430,6 +507,7 @@ export async function transcodeImageActivity(
|
||||
}
|
||||
|
||||
export interface GenerateSpriteActivityParams {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
filePath: string
|
||||
mediaInfo: PrismaJson.MediaInfo
|
||||
@@ -439,6 +517,14 @@ export interface GenerateSpriteActivityParams {
|
||||
|
||||
export async function generateSpriteActivity(params: GenerateSpriteActivityParams) {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Sprite/Poster generation cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, params.spriteSpec.key)
|
||||
await s3Service.headObject(bucket, params.posterSpec.key)
|
||||
@@ -457,6 +543,7 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
signal,
|
||||
)
|
||||
if (params.mediaInfo && params.mediaInfo.metadata) {
|
||||
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
|
||||
@@ -470,10 +557,12 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
spriteFile,
|
||||
posterFile,
|
||||
params.mediaInfo.duration,
|
||||
signal,
|
||||
)
|
||||
}
|
||||
|
||||
const spriteBuffer = fs.readFileSync(spriteFile)
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(
|
||||
bucket,
|
||||
params.spriteSpec.key,
|
||||
@@ -496,6 +585,19 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Sprite/Poster generation cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -610,6 +712,7 @@ export async function generatePdfProxyActivity(
|
||||
|
||||
const stat = fs.statSync(pdfFilePath)
|
||||
const stream = Bun.file(pdfFilePath).stream()
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(bucket, pdfProxyKey, stream, stat.size, 'application/pdf')
|
||||
|
||||
return { pdfProxyKey, pdfFilePath }
|
||||
@@ -668,9 +771,9 @@ export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityP
|
||||
include: { storageKey: true },
|
||||
})
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) {
|
||||
if (!asset || !key || asset.status === AssetStatus.pending_purge) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset not found or has no key',
|
||||
message: 'Asset not found, has no key, or is being purged',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
@@ -679,14 +782,24 @@ export async function updateAssetMediaActivity(params: UpdateAssetMediaActivityP
|
||||
const buffer = Buffer.from(JSON.stringify(params.mediaInfo))
|
||||
await s3Service.putObject(bucket, infoKey, buffer, buffer.length, 'application/json')
|
||||
|
||||
await prisma.asset.update({
|
||||
where: { id: params.assetId },
|
||||
const { count } = await prisma.asset.updateMany({
|
||||
where: {
|
||||
id: params.assetId,
|
||||
status: { not: AssetStatus.pending_purge },
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
data: { media: params.mediaInfo as any, hasJpegPreview: false },
|
||||
})
|
||||
if (count === 0) {
|
||||
logger.info(
|
||||
{ assetId: params.assetId },
|
||||
'[updateAssetMediaActivity] Asset was deleted or pending purge before media update completed',
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export async function takeScreenshotsActivity(params: {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
assetId: string
|
||||
start: number
|
||||
@@ -695,11 +808,33 @@ export async function takeScreenshotsActivity(params: {
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
}): Promise<Array<{ key: string; timestamp: number }>> {
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Screenshot extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
return await transcodeService.takeScreenshots(params)
|
||||
return await transcodeService.takeScreenshots({ ...params, signal })
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Screenshot extraction cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -720,11 +855,21 @@ export async function takeScreenshotsActivity(params: {
|
||||
export interface ExtractPosterActivityParams {
|
||||
assetKey: string
|
||||
posterSpec: PrismaJson.PosterInfo
|
||||
taskId?: string
|
||||
signal?: AbortSignal
|
||||
}
|
||||
|
||||
export async function extractPosterActivity(
|
||||
params: ExtractPosterActivityParams,
|
||||
): Promise<{ poster: PrismaJson.PosterInfo }> {
|
||||
const signal = params.signal || getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
try {
|
||||
await s3Service.headObject(bucket, params.posterSpec.key)
|
||||
@@ -758,9 +903,10 @@ export async function extractPosterActivity(
|
||||
'75',
|
||||
posterFile,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args], { signal })
|
||||
|
||||
const posterBuffer = fs.readFileSync(posterFile)
|
||||
await ensureAssetNotPurging(params.assetKey)
|
||||
await s3Service.putObject(
|
||||
bucket,
|
||||
params.posterSpec.key,
|
||||
@@ -773,6 +919,20 @@ export async function extractPosterActivity(
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Poster extraction cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -820,6 +980,7 @@ export async function overlayAnnotationsActivity(params: {
|
||||
}
|
||||
|
||||
export async function renderPdfPagesActivity(params: {
|
||||
taskId?: string
|
||||
assetKey: string
|
||||
assetId: string
|
||||
start: number
|
||||
@@ -827,11 +988,33 @@ export async function renderPdfPagesActivity(params: {
|
||||
commentTimestamp?: number | null
|
||||
annotations?: PrismaJson.AnnotationList | null
|
||||
}): Promise<Array<{ key: string; page: number }>> {
|
||||
const signal = getActivityCancellationSignal(params.taskId)
|
||||
if (signal?.aborted) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'PDF page rendering cancelled',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
return await transcodeService.renderPdfPages(params)
|
||||
return await transcodeService.renderPdfPages({ ...params, signal })
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
signal?.aborted ||
|
||||
code === 'ABORT_ERR' ||
|
||||
message.includes('aborted') ||
|
||||
lowerMsg.includes('abort')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'PDF page rendering cancelled',
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
@@ -937,7 +1120,7 @@ export async function createAutofillTaskIfEnabledActivity(
|
||||
project: true,
|
||||
},
|
||||
})
|
||||
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack') || asset.isDeleted) {
|
||||
if (!asset || (asset.type !== 'file' && asset.type !== 'version_stack')) {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@ export async function fetchAssetWithKey(workerQueue: string, assetId: string) {
|
||||
const { getAssetActivity } = getActivities()
|
||||
const asset = await executeActivity(workerQueue, getAssetActivity, assetId)
|
||||
const key = asset?.storageKey?.key
|
||||
if (!asset || !key) {
|
||||
if (!asset || !key || asset.status === 'pending_purge') {
|
||||
throw ApplicationFailure.create({
|
||||
message: 'Asset not found or has no key',
|
||||
message: 'Asset not found, has no key, or is being purged',
|
||||
nonRetryable: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ describe('renderPdfPagesWorkflow', () => {
|
||||
await renderPdfPagesWorkflow(task)
|
||||
|
||||
expect(mockActivities.renderPdfPagesActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-pdf-pages',
|
||||
assetKey: 'doc.pdf',
|
||||
assetId: 'asset-pdf',
|
||||
start: 1,
|
||||
|
||||
@@ -21,6 +21,7 @@ export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void>
|
||||
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
|
||||
|
||||
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: pdfKey,
|
||||
assetId: asset.id,
|
||||
start: payload.pdfPages.start,
|
||||
|
||||
@@ -87,6 +87,7 @@ describe('takeVideoScreenshotsWorkflow', () => {
|
||||
await takeVideoScreenshotsWorkflow(task)
|
||||
|
||||
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-screenshot',
|
||||
assetKey: 'video.mp4',
|
||||
assetId: 'asset-video',
|
||||
start: 0,
|
||||
@@ -157,6 +158,7 @@ describe('takeVideoScreenshotsWorkflow', () => {
|
||||
await takeVideoScreenshotsWorkflow(task)
|
||||
|
||||
expect(mockActivities.takeScreenshotsActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-screenshot-proxy',
|
||||
assetKey: 'proxy-1080p.mp4',
|
||||
assetId: 'asset-video-proxy',
|
||||
start: 1.2345,
|
||||
|
||||
@@ -33,6 +33,7 @@ export async function takeVideoScreenshotsWorkflow(task: WorkflowTask): Promise<
|
||||
const { takeScreenshotsActivity } = getActivities()
|
||||
|
||||
const screenshots = await executeActivity(workerQueue, takeScreenshotsActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: targetKey,
|
||||
assetId: asset.id,
|
||||
start: payload.screenshot.start,
|
||||
|
||||
@@ -304,11 +304,13 @@ describe('transcodeVideoWorkflow', () => {
|
||||
|
||||
await transcodeVideoWorkflow(task)
|
||||
|
||||
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'audio.wav',
|
||||
filePath: '/tmp/video.mp4',
|
||||
threads: 4,
|
||||
})
|
||||
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
assetKey: 'audio.wav',
|
||||
filePath: '/tmp/video.mp4',
|
||||
threads: 4,
|
||||
}),
|
||||
)
|
||||
|
||||
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-audio',
|
||||
@@ -390,6 +392,7 @@ describe('transcodeVideoWorkflow', () => {
|
||||
posterSpec: {
|
||||
key: 'files/asset-poster/poster.webp',
|
||||
},
|
||||
taskId: 'task-poster',
|
||||
})
|
||||
|
||||
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
|
||||
|
||||
@@ -68,6 +68,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
const posterResult = await executeActivity(workerQueue, extractPosterActivity, {
|
||||
assetKey: key,
|
||||
posterSpec,
|
||||
taskId: task.id,
|
||||
})
|
||||
mediaInfo.poster = posterResult.poster
|
||||
await executeActivity(workerQueue, updateAssetMediaActivity, {
|
||||
@@ -98,6 +99,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
}
|
||||
|
||||
const videoTranscode = await executeActivity(workerQueue, transcodeVideoActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
videoSpec,
|
||||
@@ -119,6 +121,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
const isAudio = mediaInfo.proxyType === 'audio'
|
||||
if (isAudio && metadata) {
|
||||
const audioTranscode = await executeActivity(workerQueue, transcodeAudioActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
threads: spec.threads,
|
||||
@@ -150,6 +153,7 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
}
|
||||
|
||||
const spriteResult = await executeActivity(workerQueue, generateSpriteActivity, {
|
||||
taskId: task.id,
|
||||
assetKey: key,
|
||||
filePath,
|
||||
spriteSpec,
|
||||
|
||||
@@ -48,6 +48,27 @@ describe('Task Activities', () => {
|
||||
expect(updated?.status).toBe(AssetStatus.processing)
|
||||
})
|
||||
|
||||
it('should NOT overwrite trashed status when asset is soft-deleted (isDeleted: true)', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: {
|
||||
name: 'trashed-video.mp4',
|
||||
storageKey: { create: { key: 'trashed-video.mp4' } },
|
||||
status: AssetStatus.trashed,
|
||||
isDeleted: true,
|
||||
type: 'file',
|
||||
},
|
||||
})
|
||||
|
||||
await updateAssetStatusActivity({
|
||||
assetId: asset.id,
|
||||
status: AssetStatus.processed,
|
||||
})
|
||||
|
||||
const updated = await prisma.asset.findUnique({ where: { id: asset.id } })
|
||||
expect(updated?.status).toBe(AssetStatus.trashed)
|
||||
expect(updated?.isDeleted).toBe(true)
|
||||
})
|
||||
|
||||
it('should update task usage', async () => {
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
@@ -69,4 +90,13 @@ describe('Task Activities', () => {
|
||||
expect(updated?.outputTokens).toBe(200)
|
||||
expect(updated?.model).toBe('gpt-4o')
|
||||
})
|
||||
|
||||
it('should not throw error when updating status of a non-existent or purged task', async () => {
|
||||
await expect(
|
||||
updateTaskStatusActivity({
|
||||
taskId: 'non-existent-task-id',
|
||||
status: WorkflowTaskStatus.failed,
|
||||
}),
|
||||
).resolves.not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -70,7 +70,7 @@ export interface UpdateTaskStatusParams {
|
||||
}
|
||||
|
||||
export async function updateTaskStatusActivity(params: UpdateTaskStatusParams): Promise<void> {
|
||||
await prisma.workflowTask.update({
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: params.taskId },
|
||||
data: {
|
||||
status: params.status,
|
||||
@@ -85,8 +85,8 @@ export interface UpdateAssetStatusParams {
|
||||
}
|
||||
|
||||
export async function updateAssetStatusActivity(params: UpdateAssetStatusParams): Promise<void> {
|
||||
await prisma.asset.update({
|
||||
where: { id: params.assetId },
|
||||
await prisma.asset.updateMany({
|
||||
where: { id: params.assetId, isDeleted: false },
|
||||
data: { status: params.status },
|
||||
})
|
||||
}
|
||||
@@ -99,7 +99,7 @@ export interface UpdateTaskUsageParams {
|
||||
}
|
||||
|
||||
export async function updateTaskUsageActivity(params: UpdateTaskUsageParams): Promise<void> {
|
||||
await prisma.workflowTask.update({
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: params.taskId },
|
||||
data: {
|
||||
inputTokens: params.inputTokens,
|
||||
@@ -120,7 +120,7 @@ export interface UpdateWorkflowTaskParams {
|
||||
}
|
||||
|
||||
export async function updateWorkflowTaskActivity(params: UpdateWorkflowTaskParams): Promise<void> {
|
||||
await prisma.workflowTask.update({
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: params.taskId },
|
||||
data: {
|
||||
...(params.status !== undefined ? { status: params.status } : {}),
|
||||
|
||||
@@ -4,3 +4,4 @@ export * from './temporal-executor'
|
||||
export * from './workflow-utils'
|
||||
export * from './workflow'
|
||||
export * from './activities/task'
|
||||
export * from './local-task-storage'
|
||||
|
||||
@@ -413,5 +413,153 @@ describe('LocalExecutor Integration Tests', () => {
|
||||
|
||||
unregisterLocalCancelHandler(taskId)
|
||||
})
|
||||
|
||||
it('clears active heartbeat timer when cancelling a task', async () => {
|
||||
let resolveChat: (value: unknown) => void = () => {}
|
||||
const chatPromise = new Promise((resolve) => {
|
||||
resolveChat = resolve
|
||||
})
|
||||
mocks.agentChat.mockImplementationOnce(() => chatPromise)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: 'test-asset-cancel',
|
||||
type: WorkflowTaskType.chat,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
// Wait briefly for submit to start and establish heartbeat
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
|
||||
// Heartbeat timer should be tracked
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const heartbeats = (executor as any).heartbeatIntervals as Map<string, Timer>
|
||||
expect(heartbeats.has(task.id)).toBe(true)
|
||||
|
||||
await executor.cancel(task.id)
|
||||
|
||||
// Timer should be cleared and removed
|
||||
expect(heartbeats.has(task.id)).toBe(false)
|
||||
|
||||
resolveChat(undefined)
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
})
|
||||
|
||||
it('handles heartbeat updateMany returning 0 when task is deleted from database', async () => {
|
||||
let resolveChat: (value: unknown) => void = () => {}
|
||||
const chatPromise = new Promise((resolve) => {
|
||||
resolveChat = resolve
|
||||
})
|
||||
mocks.agentChat.mockImplementationOnce(() => chatPromise)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: 'test-asset-deleted',
|
||||
type: WorkflowTaskType.chat,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
|
||||
// Delete the task from DB while task is running
|
||||
await prisma.workflowTask.delete({ where: { id: task.id } })
|
||||
|
||||
// UpdateMany should safely return count 0 without throwing P2025
|
||||
const { count } = await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id },
|
||||
data: { heartbeat: new Date() },
|
||||
})
|
||||
expect(count).toBe(0)
|
||||
|
||||
resolveChat(undefined)
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
})
|
||||
|
||||
it('aborts local task abort signal when task is cancelled', async () => {
|
||||
const { getLocalTaskAbortSignal } = await import('./workflow-utils')
|
||||
let resolveChat: (value: unknown) => void = () => {}
|
||||
const chatPromise = new Promise((resolve) => {
|
||||
resolveChat = resolve
|
||||
})
|
||||
mocks.agentChat.mockImplementationOnce(() => chatPromise)
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: 'test-asset-abort-signal',
|
||||
type: WorkflowTaskType.chat,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
|
||||
const signal = getLocalTaskAbortSignal(task.id)
|
||||
expect(signal).toBeDefined()
|
||||
expect(signal?.aborted).toBe(false)
|
||||
|
||||
await executor.cancel(task.id)
|
||||
|
||||
expect(signal?.aborted).toBe(true)
|
||||
|
||||
resolveChat(undefined)
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
})
|
||||
|
||||
it('cancels queued local tasks before execution when limiter is saturated', async () => {
|
||||
let resolveFirstTranscode: (value: unknown) => void = () => {}
|
||||
const firstTranscodePromise = new Promise((resolve) => {
|
||||
resolveFirstTranscode = resolve
|
||||
})
|
||||
mocks.transcodeMedia.mockImplementationOnce(() => firstTranscodePromise)
|
||||
mocks.transcodeMedia.mockImplementationOnce(() => Promise.resolve())
|
||||
|
||||
// 1. Task 1 starts and occupies the single transcode slot
|
||||
const task1 = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: 'asset-queued-cancel-1',
|
||||
type: WorkflowTaskType.transcode,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
// 2. Task 2 is submitted and queued because concurrency limit is 1
|
||||
const task2 = await prisma.workflowTask.create({
|
||||
data: {
|
||||
assetId: 'asset-queued-cancel-2',
|
||||
type: WorkflowTaskType.transcode,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
},
|
||||
})
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
|
||||
// Task 1 should be running
|
||||
expect(mocks.transcodeMedia).toHaveBeenCalledWith(expect.objectContaining({ id: task1.id }))
|
||||
// Task 2 should not have started yet
|
||||
expect(mocks.transcodeMedia).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: task2.id }),
|
||||
)
|
||||
|
||||
// 3. Cancel task 2 while it is still waiting in the queue
|
||||
await executor.cancel(task2.id)
|
||||
|
||||
// 4. Release task 1 so the concurrency slot becomes available
|
||||
resolveFirstTranscode(undefined)
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
|
||||
// 5. Verify task 2's workflow was NEVER executed
|
||||
expect(mocks.transcodeMedia).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: task2.id }),
|
||||
)
|
||||
|
||||
// 6. Verify task 2 was marked failed / cancelled
|
||||
const dbTask2 = await prisma.workflowTask.findUnique({
|
||||
where: { id: task2.id },
|
||||
})
|
||||
expect(dbTask2?.status).toBe(WorkflowTaskStatus.failed)
|
||||
expect((dbTask2?.output as { error?: string })?.error).toContain('cancelled')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,8 +2,15 @@ import { prisma } from '@shumai/db'
|
||||
import { WorkflowTask, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { Executor } from './executor'
|
||||
import * as taskActivities from './activities/task'
|
||||
import {
|
||||
getConcurrencyLimit,
|
||||
registerLocalTaskAbortController,
|
||||
runInLocalTaskContext,
|
||||
triggerLocalCancel,
|
||||
unregisterLocalTaskAbortController,
|
||||
} from './workflow-utils'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { getConcurrencyLimit, triggerLocalCancel } from './workflow-utils'
|
||||
import './local-task-storage'
|
||||
|
||||
type WorkflowFn = (task: WorkflowTask) => Promise<void>
|
||||
|
||||
@@ -30,6 +37,7 @@ export function registerActivities(acts: Record<string, unknown>) {
|
||||
globalObj.__localActivities = activityRegistry
|
||||
globalObj.__localLogger = logger
|
||||
|
||||
// ConcurrencyLimiter manages concurrent execution of promises with a FIFO queue.
|
||||
export class ConcurrencyLimiter {
|
||||
private activeCount = 0
|
||||
private queue: (() => void)[] = []
|
||||
@@ -72,6 +80,7 @@ export class ConcurrencyLimiter {
|
||||
export class LocalExecutor implements Executor {
|
||||
private interval: Timer | null = null
|
||||
private processingTasks = new Set<string>()
|
||||
private heartbeatIntervals = new Map<string, Timer>()
|
||||
|
||||
private transcodeLimiter: ConcurrencyLimiter
|
||||
private generalLimiter: ConcurrencyLimiter
|
||||
@@ -94,7 +103,7 @@ export class LocalExecutor implements Executor {
|
||||
|
||||
async submit(task: WorkflowTask): Promise<string> {
|
||||
if (task.status !== WorkflowTaskStatus.pending) {
|
||||
await prisma.workflowTask.update({
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id },
|
||||
data: { status: WorkflowTaskStatus.pending },
|
||||
})
|
||||
@@ -135,14 +144,26 @@ export class LocalExecutor implements Executor {
|
||||
// 2. Start heartbeat updater immediately
|
||||
const heartbeatInterval = setInterval(async () => {
|
||||
try {
|
||||
await prisma.workflowTask.update({
|
||||
const { count } = await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id },
|
||||
data: { heartbeat: new Date() },
|
||||
})
|
||||
if (count === 0) {
|
||||
clearInterval(heartbeatInterval)
|
||||
this.heartbeatIntervals.delete(task.id)
|
||||
logger.debug(
|
||||
{ taskId: task.id },
|
||||
'[LocalExecutor] Stopped heartbeat for deleted or cancelled task',
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[LocalExecutor] Failed to update heartbeat for task ${task.id}:`, err)
|
||||
}
|
||||
}, 5000)
|
||||
this.heartbeatIntervals.set(task.id, heartbeatInterval)
|
||||
|
||||
const abortController = new AbortController()
|
||||
registerLocalTaskAbortController(task.id, abortController)
|
||||
|
||||
const limiter =
|
||||
task.type && task.type.startsWith('transcode')
|
||||
@@ -152,10 +173,32 @@ export class LocalExecutor implements Executor {
|
||||
limiter
|
||||
.run(async () => {
|
||||
try {
|
||||
await this.processTaskWrapper(task)
|
||||
if (abortController.signal.aborted) {
|
||||
logger.info(
|
||||
{ taskId: task.id },
|
||||
'[LocalExecutor] Queued task was aborted before execution',
|
||||
)
|
||||
try {
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id, status: WorkflowTaskStatus.processing },
|
||||
data: {
|
||||
status: WorkflowTaskStatus.failed,
|
||||
output: { error: 'Workflow task was cancelled before execution' },
|
||||
},
|
||||
})
|
||||
} catch {
|
||||
// Ignore if task was already deleted
|
||||
}
|
||||
return
|
||||
}
|
||||
await runInLocalTaskContext({ taskId: task.id, abortController }, async () => {
|
||||
await this.processTaskWrapper(task)
|
||||
})
|
||||
} finally {
|
||||
// 3. Clean up the heartbeat updater and processing set when the task finishes
|
||||
clearInterval(heartbeatInterval)
|
||||
this.heartbeatIntervals.delete(task.id)
|
||||
unregisterLocalTaskAbortController(task.id)
|
||||
this.processingTasks.delete(task.id)
|
||||
}
|
||||
})
|
||||
@@ -168,7 +211,13 @@ export class LocalExecutor implements Executor {
|
||||
}
|
||||
|
||||
async cancel(taskId: string): Promise<void> {
|
||||
const timer = this.heartbeatIntervals.get(taskId)
|
||||
if (timer) {
|
||||
clearInterval(timer)
|
||||
this.heartbeatIntervals.delete(taskId)
|
||||
}
|
||||
triggerLocalCancel(taskId)
|
||||
logger.info({ taskId }, '[LocalExecutor] Cancelled workflow task')
|
||||
}
|
||||
|
||||
start(): void {
|
||||
@@ -181,6 +230,13 @@ export class LocalExecutor implements Executor {
|
||||
clearInterval(this.interval)
|
||||
this.interval = null
|
||||
}
|
||||
for (const timer of this.heartbeatIntervals.values()) {
|
||||
clearInterval(timer)
|
||||
}
|
||||
this.heartbeatIntervals.clear()
|
||||
for (const taskId of this.processingTasks) {
|
||||
triggerLocalCancel(taskId)
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the array of task promises so tests can await them to avoid early rollback.
|
||||
@@ -188,6 +244,10 @@ export class LocalExecutor implements Executor {
|
||||
const promises: Promise<void>[] = []
|
||||
try {
|
||||
const staleTime = new Date(Date.now() - 30 * 1000)
|
||||
|
||||
// Find tasks that are:
|
||||
// 1. Pending (new tasks)
|
||||
// 2. Processing but stale (worker crashed, heartbeat > 30s ago)
|
||||
const tasks = await prisma.workflowTask.findMany({
|
||||
where: {
|
||||
OR: [
|
||||
@@ -198,6 +258,7 @@ export class LocalExecutor implements Executor {
|
||||
},
|
||||
],
|
||||
},
|
||||
orderBy: { id: 'asc' },
|
||||
take: 50,
|
||||
})
|
||||
|
||||
@@ -206,8 +267,7 @@ export class LocalExecutor implements Executor {
|
||||
this.processingTasks.add(task.id)
|
||||
|
||||
// 1. Immediately mark task as processing and set initial heartbeat in DB
|
||||
// to guarantee no other ticks/replicas can double-query it.
|
||||
// We use updateMany to atomically claim it only if it is still pending or stale.
|
||||
// atomically, ensuring no two workers pick up the same task.
|
||||
try {
|
||||
const affected = await prisma.workflowTask.updateMany({
|
||||
where: {
|
||||
@@ -230,23 +290,38 @@ export class LocalExecutor implements Executor {
|
||||
continue
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[LocalExecutor] Failed to mark task ${task.id} as processing:`, err)
|
||||
console.error(
|
||||
`[LocalExecutor] Failed to mark task ${task.id} as processing in tick:`,
|
||||
err,
|
||||
)
|
||||
this.processingTasks.delete(task.id)
|
||||
continue
|
||||
}
|
||||
|
||||
// 2. Start heartbeat updater immediately so it updates heartbeat
|
||||
// 2. Start heartbeat updater immediately to prevent other workers from considering it stale
|
||||
// even while the task is queued waiting for limiter capacity.
|
||||
const heartbeatInterval = setInterval(async () => {
|
||||
try {
|
||||
await prisma.workflowTask.update({
|
||||
const { count } = await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id },
|
||||
data: { heartbeat: new Date() },
|
||||
})
|
||||
if (count === 0) {
|
||||
clearInterval(heartbeatInterval)
|
||||
this.heartbeatIntervals.delete(task.id)
|
||||
logger.debug(
|
||||
{ taskId: task.id },
|
||||
'[LocalExecutor] Stopped heartbeat for deleted or cancelled task',
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[LocalExecutor] Failed to update heartbeat for task ${task.id}:`, err)
|
||||
}
|
||||
}, 5000)
|
||||
this.heartbeatIntervals.set(task.id, heartbeatInterval)
|
||||
|
||||
const abortController = new AbortController()
|
||||
registerLocalTaskAbortController(task.id, abortController)
|
||||
|
||||
const limiter =
|
||||
task.type && task.type.startsWith('transcode')
|
||||
@@ -256,10 +331,32 @@ export class LocalExecutor implements Executor {
|
||||
const promise = limiter
|
||||
.run(async () => {
|
||||
try {
|
||||
await this.processTaskWrapper(task)
|
||||
if (abortController.signal.aborted) {
|
||||
logger.info(
|
||||
{ taskId: task.id },
|
||||
'[LocalExecutor] Queued task was aborted before execution',
|
||||
)
|
||||
try {
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id, status: WorkflowTaskStatus.processing },
|
||||
data: {
|
||||
status: WorkflowTaskStatus.failed,
|
||||
output: { error: 'Workflow task was cancelled before execution' },
|
||||
},
|
||||
})
|
||||
} catch {
|
||||
// Ignore if task was already deleted
|
||||
}
|
||||
return
|
||||
}
|
||||
await runInLocalTaskContext({ taskId: task.id, abortController }, async () => {
|
||||
await this.processTaskWrapper(task)
|
||||
})
|
||||
} finally {
|
||||
// 3. Clean up the heartbeat updater and processing set when the task finishes
|
||||
clearInterval(heartbeatInterval)
|
||||
this.heartbeatIntervals.delete(task.id)
|
||||
unregisterLocalTaskAbortController(task.id)
|
||||
this.processingTasks.delete(task.id)
|
||||
}
|
||||
})
|
||||
@@ -285,7 +382,7 @@ export class LocalExecutor implements Executor {
|
||||
} catch (err) {
|
||||
console.error(`Error in workflow ${task.type} for task ${task.id}:`, err)
|
||||
try {
|
||||
await prisma.workflowTask.update({
|
||||
await prisma.workflowTask.updateMany({
|
||||
where: { id: task.id },
|
||||
data: {
|
||||
status: WorkflowTaskStatus.failed,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import type { LocalTaskContext } from './workflow-utils'
|
||||
|
||||
// Store AsyncLocalStorage on globalThis to share with workflow-utils without
|
||||
// requiring workflow-utils to import node:async_hooks (which would break Temporal's Webpack workflow bundler).
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const globalObj = globalThis as any
|
||||
if (!globalObj.__localTaskStorage) {
|
||||
globalObj.__localTaskStorage = new AsyncLocalStorage<LocalTaskContext>()
|
||||
}
|
||||
|
||||
export const localTaskStorage: AsyncLocalStorage<LocalTaskContext> = globalObj.__localTaskStorage
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Connection, Client } from '@temporalio/client'
|
||||
import { WorkflowTask, WorkflowTaskType } from '@shumai/db'
|
||||
import { prisma, WorkflowTask, WorkflowTaskType } from '@shumai/db'
|
||||
import { logger } from '@shumai/core/src/logger'
|
||||
import { Executor } from './executor'
|
||||
import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils'
|
||||
|
||||
@@ -80,9 +81,18 @@ export class TemporalExecutor implements Executor {
|
||||
|
||||
async cancel(taskId: string): Promise<void> {
|
||||
const client = await this.getClient()
|
||||
const workflowId = `chat-${taskId}`
|
||||
const handle = client.workflow.getHandle(workflowId)
|
||||
await handle.cancel()
|
||||
const task = await prisma.workflowTask.findUnique({
|
||||
where: { id: taskId },
|
||||
select: { type: true },
|
||||
})
|
||||
const workflowId = task?.type ? `${task.type}-${taskId}` : `chat-${taskId}`
|
||||
try {
|
||||
const handle = client.workflow.getHandle(workflowId)
|
||||
await handle.cancel()
|
||||
logger.info({ taskId, workflowId }, '[TemporalExecutor] Cancelled workflow task')
|
||||
} catch (err) {
|
||||
logger.warn({ taskId, workflowId, err }, '[TemporalExecutor] Failed to cancel workflow task')
|
||||
}
|
||||
}
|
||||
|
||||
start(): void {
|
||||
|
||||
@@ -204,6 +204,11 @@ export function getConcurrencyLimit(envVar: string | undefined, defaultValue: nu
|
||||
return parsed
|
||||
}
|
||||
|
||||
export interface LocalTaskContext {
|
||||
taskId: string
|
||||
abortController: AbortController
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const globalObjWithCancel = globalThis as any
|
||||
if (!globalObjWithCancel.__localCancelHandlers) {
|
||||
@@ -211,6 +216,52 @@ if (!globalObjWithCancel.__localCancelHandlers) {
|
||||
}
|
||||
const localCancelHandlers: Map<string, () => void> = globalObjWithCancel.__localCancelHandlers
|
||||
|
||||
if (!globalObjWithCancel.__localTaskAbortControllers) {
|
||||
globalObjWithCancel.__localTaskAbortControllers = new Map<string, AbortController>()
|
||||
}
|
||||
const localTaskAbortControllers: Map<string, AbortController> =
|
||||
globalObjWithCancel.__localTaskAbortControllers
|
||||
|
||||
interface LocalTaskStorageHolder {
|
||||
getStore: () => LocalTaskContext | undefined
|
||||
run: <T>(store: LocalTaskContext, fn: () => T) => T
|
||||
}
|
||||
|
||||
function getLocalTaskStorage(): LocalTaskStorageHolder | undefined {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return (globalThis as any).__localTaskStorage
|
||||
}
|
||||
|
||||
export function registerLocalTaskAbortController(
|
||||
taskId: string,
|
||||
controller: AbortController,
|
||||
): void {
|
||||
localTaskAbortControllers.set(taskId, controller)
|
||||
}
|
||||
|
||||
export function unregisterLocalTaskAbortController(taskId: string): void {
|
||||
localTaskAbortControllers.delete(taskId)
|
||||
}
|
||||
|
||||
export function runInLocalTaskContext<T>(context: LocalTaskContext, fn: () => T): T {
|
||||
registerLocalTaskAbortController(context.taskId, context.abortController)
|
||||
const storage = getLocalTaskStorage()
|
||||
if (storage) {
|
||||
return storage.run(context, fn)
|
||||
}
|
||||
return fn()
|
||||
}
|
||||
|
||||
export function getLocalTaskAbortSignal(taskId?: string): AbortSignal | undefined {
|
||||
if (taskId) {
|
||||
const controller = localTaskAbortControllers.get(taskId)
|
||||
if (controller) {
|
||||
return controller.signal
|
||||
}
|
||||
}
|
||||
return getLocalTaskStorage()?.getStore()?.abortController?.signal
|
||||
}
|
||||
|
||||
export function registerLocalCancelHandler(taskId: string, cancelFn: () => void) {
|
||||
localCancelHandlers.set(taskId, cancelFn)
|
||||
}
|
||||
@@ -220,10 +271,17 @@ export function unregisterLocalCancelHandler(taskId: string) {
|
||||
}
|
||||
|
||||
export function triggerLocalCancel(taskId: string): boolean {
|
||||
let cancelled = false
|
||||
const controller = localTaskAbortControllers.get(taskId)
|
||||
if (controller) {
|
||||
controller.abort()
|
||||
localTaskAbortControllers.delete(taskId)
|
||||
cancelled = true
|
||||
}
|
||||
const handler = localCancelHandlers.get(taskId)
|
||||
if (handler) {
|
||||
handler()
|
||||
return true
|
||||
cancelled = true
|
||||
}
|
||||
return false
|
||||
return cancelled
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { prisma, registerWorkflowTrigger, WorkflowTask } from '@shumai/db'
|
||||
import { prisma, registerWorkflowCancel, registerWorkflowTrigger, WorkflowTask } from '@shumai/db'
|
||||
import path from 'path'
|
||||
import { ulid } from 'ulid'
|
||||
import * as taskActivities from './activities/task'
|
||||
@@ -187,3 +187,6 @@ export const workflowService = new WorkflowService()
|
||||
registerWorkflowTrigger(async (task) => {
|
||||
await workflowService.submit(task)
|
||||
})
|
||||
registerWorkflowCancel(async (taskId) => {
|
||||
await workflowService.cancel(taskId)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user