feat(transcode): add PDF preview, sprite generation, and annotation support

This commit is contained in:
yiling
2026-07-18 10:32:38 +08:00
parent 3383079787
commit bc1491bed5
31 changed files with 928 additions and 46 deletions
@@ -3,7 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { prisma } from '@shumai/db'
import { setupTestDbHooks } from '@shumai/db/test'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { transcodeService } from '@shumai/core'
import { transcodeService } from '@shumai/core/src/transcode/transcode'
import * as child_process from 'child_process'
vi.mock('child_process', () => ({
@@ -39,10 +39,25 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({
getVideoInfo: vi.fn(),
getAudioInfo: vi.fn(),
getImageInfo: vi.fn(),
getPdfInfo: vi.fn().mockResolvedValue({
originalWidth: 800,
originalHeight: 1000,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: 10,
hasAudio: false,
mimeType: 'application/pdf',
}),
transcodeVideo: vi.fn(),
transcodeAudio: vi.fn(),
transcodeImage: vi.fn(),
generateSprite: vi.fn(),
generatePdfSprite: vi.fn().mockResolvedValue({
pageCount: 10,
originalWidth: 800,
originalHeight: 1000,
}),
createTempDir: vi.fn().mockReturnValue('/tmp'),
removeDir: vi.fn(),
takeScreenshots: vi.fn(),
+40 -7
View File
@@ -1,6 +1,6 @@
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core'
import { transcodeService } from '@shumai/core/src/transcode/transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { stemFromKey } from '@shumai/core/src/utils/filename'
import { ApplicationFailure } from '@temporalio/activity'
@@ -163,6 +163,25 @@ export async function getMediaInfoActivity(params: {
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
)
} else if (params.mediaType === 'application/pdf') {
const info = await transcodeService.getPdfInfo(params.filePath)
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
originalWidth: info.originalWidth,
originalHeight: info.originalHeight,
duration: 0,
bitRate: 0,
frameRate: 0,
totalFrames: info.totalFrames,
startTimecode: '00:00:00:00',
hasAudio: false,
format: {},
}
metadataUpdates.push(
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
{ key: 'total_pages', value: info.totalFrames },
)
}
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
@@ -401,12 +420,26 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
const posterFile = path.join(tmpDir, 'poster.webp')
try {
await transcodeService.generateSprite(
params.filePath,
spriteFile,
posterFile,
params.mediaInfo.duration,
)
if (params.mediaInfo.mimeType === 'application/pdf') {
const pdfRes = await transcodeService.generatePdfSprite(
params.filePath,
spriteFile,
posterFile,
)
if (params.mediaInfo && params.mediaInfo.metadata) {
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
params.mediaInfo.frames = pdfRes.pageCount
}
} else {
await transcodeService.generateSprite(
params.filePath,
spriteFile,
posterFile,
params.mediaInfo.duration,
)
}
const spriteBuffer = fs.readFileSync(spriteFile)
await s3Service.putObject(
+38
View File
@@ -82,3 +82,41 @@ export class ImageTranscoder {
return task.id
}
}
export class PdfTranscoder {
private spec: PrismaJson.TaskSpec = {}
constructor(
private readonly db: TransactionClient,
private readonly assetId: string,
private readonly teamId: string,
private readonly projectId: string,
) {}
withSprite(): this {
this.spec.sprite = true
return this
}
withPoster(): this {
this.spec.poster = true
return this
}
async submit(): Promise<string> {
const task = await this.db.workflowTask.create({
data: {
assetId: this.assetId,
teamId: this.teamId,
projectId: this.projectId,
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
payload: {
projectId: this.projectId,
transcode: this.spec,
},
},
})
return task.id
}
}
@@ -245,6 +245,89 @@ describe('Transcode Workflow', () => {
})
})
it('should process pdf transcode and sprite generation successfully', async () => {
const task: WorkflowTask = {
id: 'task-pdf',
assetId: 'asset-pdf',
type: WorkflowTaskType.transcode,
status: WorkflowTaskStatus.pending,
sessionId: null,
output: null,
payload: {
projectId: 'proj-1',
transcode: {
sprite: true,
poster: true,
},
},
createdAt: new Date(),
updatedAt: new Date(),
heartbeat: null,
teamId: 'team-1',
projectId: 'proj-1',
uid: 'task-uid-pdf',
model: null,
inputTokens: 0,
outputTokens: 0,
}
mockActivities.getAssetActivity.mockResolvedValue({
id: 'asset-pdf',
storageKey: { key: 'document.pdf' },
mediaType: 'application/pdf',
})
mockActivities.getMediaInfoActivity.mockResolvedValue({
mimeType: 'application/pdf',
metadata: {
originalWidth: 800,
originalHeight: 1000,
duration: 0,
frameRate: 0,
totalFrames: 12,
startTimecode: '00:00:00:00',
bitRate: 0,
hasAudio: false,
format: {},
},
videoTranscodes: [],
imageTranscodes: [],
})
mockActivities.generateSpriteActivity.mockResolvedValue({
sprite: { key: 'sprite.webp', frames: 100, tileX: 10, tileY: 10 },
poster: { key: 'poster.webp' },
})
await transcodeMedia(task)
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
status: AssetStatus.processing,
})
expect(mockActivities.generateSpriteActivity).toHaveBeenCalledWith({
assetKey: 'document.pdf',
filePath: '/tmp/video.mp4',
spriteSpec: expect.objectContaining({ key: 'sprite.webp' }),
posterSpec: expect.objectContaining({ key: 'poster.webp' }),
mediaInfo: expect.objectContaining({ mimeType: 'application/pdf' }),
})
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
mediaInfo: expect.objectContaining({
sprite: { key: 'sprite.webp', frames: 100, tileX: 10, tileY: 10 },
poster: { key: 'poster.webp' },
}),
})
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
assetId: 'asset-pdf',
status: AssetStatus.processed,
})
})
it('should handle failures and update task status with error', async () => {
const task: WorkflowTask = {
id: 'task-fail',