mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 09:57:20 +00:00
feat(transcode): add PDF preview, sprite generation, and annotation support
This commit is contained in:
@@ -3,7 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { setupTestDbHooks } from '@shumai/db/test'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import * as child_process from 'child_process'
|
||||
|
||||
vi.mock('child_process', () => ({
|
||||
@@ -39,10 +39,25 @@ vi.mock('@shumai/core/src/transcode/transcode', () => ({
|
||||
getVideoInfo: vi.fn(),
|
||||
getAudioInfo: vi.fn(),
|
||||
getImageInfo: vi.fn(),
|
||||
getPdfInfo: vi.fn().mockResolvedValue({
|
||||
originalWidth: 800,
|
||||
originalHeight: 1000,
|
||||
duration: 0,
|
||||
bitRate: 0,
|
||||
frameRate: 0,
|
||||
totalFrames: 10,
|
||||
hasAudio: false,
|
||||
mimeType: 'application/pdf',
|
||||
}),
|
||||
transcodeVideo: vi.fn(),
|
||||
transcodeAudio: vi.fn(),
|
||||
transcodeImage: vi.fn(),
|
||||
generateSprite: vi.fn(),
|
||||
generatePdfSprite: vi.fn().mockResolvedValue({
|
||||
pageCount: 10,
|
||||
originalWidth: 800,
|
||||
originalHeight: 1000,
|
||||
}),
|
||||
createTempDir: vi.fn().mockReturnValue('/tmp'),
|
||||
removeDir: vi.fn(),
|
||||
takeScreenshots: vi.fn(),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { prisma, WorkflowTaskType, WorkflowTaskStatus } from '@shumai/db'
|
||||
import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { stemFromKey } from '@shumai/core/src/utils/filename'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
@@ -163,6 +163,25 @@ export async function getMediaInfoActivity(params: {
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
)
|
||||
} else if (params.mediaType === 'application/pdf') {
|
||||
const info = await transcodeService.getPdfInfo(params.filePath)
|
||||
mediaInfo.frames = info.totalFrames
|
||||
mediaInfo.metadata = {
|
||||
originalWidth: info.originalWidth,
|
||||
originalHeight: info.originalHeight,
|
||||
duration: 0,
|
||||
bitRate: 0,
|
||||
frameRate: 0,
|
||||
totalFrames: info.totalFrames,
|
||||
startTimecode: '00:00:00:00',
|
||||
hasAudio: false,
|
||||
format: {},
|
||||
}
|
||||
metadataUpdates.push(
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
{ key: 'total_pages', value: info.totalFrames },
|
||||
)
|
||||
}
|
||||
|
||||
await metadataService.updateAssetMetadata(params.assetId, metadataUpdates, true)
|
||||
@@ -401,12 +420,26 @@ export async function generateSpriteActivity(params: GenerateSpriteActivityParam
|
||||
const posterFile = path.join(tmpDir, 'poster.webp')
|
||||
|
||||
try {
|
||||
await transcodeService.generateSprite(
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
params.mediaInfo.duration,
|
||||
)
|
||||
if (params.mediaInfo.mimeType === 'application/pdf') {
|
||||
const pdfRes = await transcodeService.generatePdfSprite(
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
)
|
||||
if (params.mediaInfo && params.mediaInfo.metadata) {
|
||||
params.mediaInfo.metadata.totalFrames = pdfRes.pageCount
|
||||
params.mediaInfo.metadata.originalWidth = pdfRes.originalWidth
|
||||
params.mediaInfo.metadata.originalHeight = pdfRes.originalHeight
|
||||
params.mediaInfo.frames = pdfRes.pageCount
|
||||
}
|
||||
} else {
|
||||
await transcodeService.generateSprite(
|
||||
params.filePath,
|
||||
spriteFile,
|
||||
posterFile,
|
||||
params.mediaInfo.duration,
|
||||
)
|
||||
}
|
||||
|
||||
const spriteBuffer = fs.readFileSync(spriteFile)
|
||||
await s3Service.putObject(
|
||||
|
||||
@@ -82,3 +82,41 @@ export class ImageTranscoder {
|
||||
return task.id
|
||||
}
|
||||
}
|
||||
|
||||
export class PdfTranscoder {
|
||||
private spec: PrismaJson.TaskSpec = {}
|
||||
|
||||
constructor(
|
||||
private readonly db: TransactionClient,
|
||||
private readonly assetId: string,
|
||||
private readonly teamId: string,
|
||||
private readonly projectId: string,
|
||||
) {}
|
||||
|
||||
withSprite(): this {
|
||||
this.spec.sprite = true
|
||||
return this
|
||||
}
|
||||
|
||||
withPoster(): this {
|
||||
this.spec.poster = true
|
||||
return this
|
||||
}
|
||||
|
||||
async submit(): Promise<string> {
|
||||
const task = await this.db.workflowTask.create({
|
||||
data: {
|
||||
assetId: this.assetId,
|
||||
teamId: this.teamId,
|
||||
projectId: this.projectId,
|
||||
type: WorkflowTaskType.transcode,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
payload: {
|
||||
projectId: this.projectId,
|
||||
transcode: this.spec,
|
||||
},
|
||||
},
|
||||
})
|
||||
return task.id
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,6 +245,89 @@ describe('Transcode Workflow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('should process pdf transcode and sprite generation successfully', async () => {
|
||||
const task: WorkflowTask = {
|
||||
id: 'task-pdf',
|
||||
assetId: 'asset-pdf',
|
||||
type: WorkflowTaskType.transcode,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
sessionId: null,
|
||||
output: null,
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
transcode: {
|
||||
sprite: true,
|
||||
poster: true,
|
||||
},
|
||||
},
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
heartbeat: null,
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
uid: 'task-uid-pdf',
|
||||
model: null,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-pdf',
|
||||
storageKey: { key: 'document.pdf' },
|
||||
mediaType: 'application/pdf',
|
||||
})
|
||||
|
||||
mockActivities.getMediaInfoActivity.mockResolvedValue({
|
||||
mimeType: 'application/pdf',
|
||||
metadata: {
|
||||
originalWidth: 800,
|
||||
originalHeight: 1000,
|
||||
duration: 0,
|
||||
frameRate: 0,
|
||||
totalFrames: 12,
|
||||
startTimecode: '00:00:00:00',
|
||||
bitRate: 0,
|
||||
hasAudio: false,
|
||||
format: {},
|
||||
},
|
||||
videoTranscodes: [],
|
||||
imageTranscodes: [],
|
||||
})
|
||||
|
||||
mockActivities.generateSpriteActivity.mockResolvedValue({
|
||||
sprite: { key: 'sprite.webp', frames: 100, tileX: 10, tileY: 10 },
|
||||
poster: { key: 'poster.webp' },
|
||||
})
|
||||
|
||||
await transcodeMedia(task)
|
||||
|
||||
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-pdf',
|
||||
status: AssetStatus.processing,
|
||||
})
|
||||
|
||||
expect(mockActivities.generateSpriteActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'document.pdf',
|
||||
filePath: '/tmp/video.mp4',
|
||||
spriteSpec: expect.objectContaining({ key: 'sprite.webp' }),
|
||||
posterSpec: expect.objectContaining({ key: 'poster.webp' }),
|
||||
mediaInfo: expect.objectContaining({ mimeType: 'application/pdf' }),
|
||||
})
|
||||
|
||||
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-pdf',
|
||||
mediaInfo: expect.objectContaining({
|
||||
sprite: { key: 'sprite.webp', frames: 100, tileX: 10, tileY: 10 },
|
||||
poster: { key: 'poster.webp' },
|
||||
}),
|
||||
})
|
||||
|
||||
expect(mockActivities.updateAssetStatusActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-pdf',
|
||||
status: AssetStatus.processed,
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle failures and update task status with error', async () => {
|
||||
const task: WorkflowTask = {
|
||||
id: 'task-fail',
|
||||
|
||||
Reference in New Issue
Block a user