From f158d0468602ee6c2de129074be3e2b62c94d7ee Mon Sep 17 00:00:00 2001 From: yiling Date: Mon, 20 Jul 2026 10:22:36 +0800 Subject: [PATCH] feat(asset): replace mediaType/mimeType logic with proxyType --- packages/agent/src/activities/ai.ts | 5 +- .../src/database-session-storage.test.ts | 2 + packages/agent/src/index.ts | 17 ++++--- packages/agent/src/tools/read-pdf-pages.ts | 8 ++++ .../src/workflows/agent-autofill.test.ts | 2 + .../agent/src/workflows/agent-autofill.ts | 5 +- .../agent-chat-prompt-builder.test.ts | 17 +++++-- .../workflows/agent-chat-prompt-builder.ts | 37 ++++++--------- .../agent/src/workflows/agent-chat.test.ts | 10 ++-- packages/agent/src/workflows/agent-chat.ts | 6 ++- packages/core/src/asset/asset.ts | 13 +++-- packages/core/src/chat/chat.test.ts | 1 + packages/core/src/chat/chat.ts | 3 +- .../core/src/notification/notification.ts | 1 + packages/core/src/upload/upload.ts | 21 ++++----- packages/core/src/utils/mime.test.ts | 30 +++++++++++- packages/core/src/utils/mime.ts | 37 +++++++++++++++ packages/dtos/src/asset.ts | 1 + packages/dtos/src/notification.ts | 1 + packages/e2e/workflow/agent-autofill.test.ts | 1 + .../src/activities/transcode.test.ts | 4 ++ .../transcode/src/activities/transcode.ts | 47 +++++++------------ .../src/workflows/render-pdf-pages.ts | 3 +- .../src/workflows/transcode-image.ts | 1 + .../transcode/src/workflows/transcode-pdf.ts | 1 + .../src/workflows/transcode-video.ts | 2 + packages/webui/components/breadcrumb-nav.tsx | 3 +- .../webui/components/chat/message-card.tsx | 2 +- .../components/compare/compare-utils.test.ts | 10 ++-- .../webui/components/compare/compare-utils.ts | 9 ++-- .../file-browser/file-list-item.tsx | 3 +- .../components/file-viewer-left-sidebar.tsx | 3 +- .../components/file-viewer-right-sidebar.tsx | 2 +- .../webui/components/notification-card.tsx | 5 +- .../webui/components/public-share-manager.tsx | 7 ++- .../viewers/video/compare-video-pane.tsx | 4 +- .../viewers/video/video-control-bar.tsx | 2 +- .../components/viewers/video/video-viewer.tsx | 4 +- packages/webui/e2e/harness/fixture.ts | 6 +++ .../$projectId/files/$fileId.lazy.tsx | 2 +- 40 files changed, 219 insertions(+), 119 deletions(-) diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index 9ccc19a2..27728cd9 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -88,8 +88,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams) outputTokens: 0, } - const isImage = asset.mediaType.startsWith('image/') - const isVideo = asset.mediaType.startsWith('video/') + const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType + const isImage = proxyType === 'image' + const isVideo = proxyType === 'video' if (!isImage && !isVideo) { throw ApplicationFailure.create({ diff --git a/packages/agent/src/database-session-storage.test.ts b/packages/agent/src/database-session-storage.test.ts index eb98708e..51fea324 100644 --- a/packages/agent/src/database-session-storage.test.ts +++ b/packages/agent/src/database-session-storage.test.ts @@ -409,6 +409,7 @@ describe('DatabaseSessionStorage', () => { name: 'test.png', type: 'file', mediaType: 'image/png', + media: { proxyType: 'image' }, projectId: project.id, status: 'uploaded', }, @@ -461,6 +462,7 @@ describe('DatabaseSessionStorage', () => { name: 'test.mp4', type: 'file', mediaType: 'video/mp4', + media: { proxyType: 'video' }, projectId: project.id, status: 'uploaded', }, diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 12c9f02f..d300ba87 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -235,15 +235,14 @@ export async function createAgentSession(params: CreateAgentSessionParams) { const asset = await prisma.asset.findUnique({ where: { id: assetId }, }) - if (asset?.mediaType) { - const type = getSimpleMediaType(asset.mediaType) - if (type === 'image') { - mediaTools.push(createAnalyzeImageTool(assetId, userCommentId)) - } else if (type === 'video') { - mediaTools.push(createScreenshotTool(assetId, userCommentId)) - } else if (type === 'pdf') { - mediaTools.push(createReadPdfPagesTool(assetId, userCommentId)) - } + const proxyType = (asset?.media as PrismaJson.MediaInfo | null)?.proxyType + const type = getSimpleMediaType(proxyType) + if (type === 'image') { + mediaTools.push(createAnalyzeImageTool(assetId, userCommentId)) + } else if (type === 'video') { + mediaTools.push(createScreenshotTool(assetId, userCommentId)) + } else if (type === 'pdf') { + mediaTools.push(createReadPdfPagesTool(assetId, userCommentId)) } } diff --git a/packages/agent/src/tools/read-pdf-pages.ts b/packages/agent/src/tools/read-pdf-pages.ts index e64620ec..a4d7a056 100644 --- a/packages/agent/src/tools/read-pdf-pages.ts +++ b/packages/agent/src/tools/read-pdf-pages.ts @@ -77,6 +77,14 @@ export function createReadPdfPagesTool( } } + const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType + if (proxyType && proxyType !== 'pdf') { + return { + content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }], + details: {}, + } + } + // Fetch trigger comment details for comment page and annotations let commentTimestamp: number | null = null let annotations: unknown = null diff --git a/packages/agent/src/workflows/agent-autofill.test.ts b/packages/agent/src/workflows/agent-autofill.test.ts index c4fb185f..ae344d55 100644 --- a/packages/agent/src/workflows/agent-autofill.test.ts +++ b/packages/agent/src/workflows/agent-autofill.test.ts @@ -64,6 +64,7 @@ describe('Agent Autofill Workflow', () => { storageKey: { key: 'asset-key' }, project: { id: 'p1', teamId: 't1' }, mediaType: 'image/png', + media: { proxyType: 'image' }, }) mockActivities.downloadMediaToTmpActivity.mockResolvedValue({ filePath: '/tmp/test.png', @@ -290,6 +291,7 @@ describe('Agent Autofill Workflow', () => { storageKey: { key: 'asset-key' }, project: { id: 'p1', teamId: 't1' }, mediaType: 'audio/mp3', + media: { proxyType: 'audio' }, }) const task = await prisma.workflowTask.create({ diff --git a/packages/agent/src/workflows/agent-autofill.ts b/packages/agent/src/workflows/agent-autofill.ts index 84e620f8..c2d48186 100644 --- a/packages/agent/src/workflows/agent-autofill.ts +++ b/packages/agent/src/workflows/agent-autofill.ts @@ -64,8 +64,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { const projectId = asset.project.id // 2. Prepare Data (Images) - const isImage = asset.mediaType?.startsWith('image/') || false - const isVideo = asset.mediaType?.startsWith('video/') || false + const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType + const isImage = proxyType === 'image' + const isVideo = proxyType === 'video' if (!isImage && !isVideo) { throw ApplicationFailure.create({ diff --git a/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts b/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts index 11730672..159c5efe 100644 --- a/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts +++ b/packages/agent/src/workflows/agent-chat-prompt-builder.test.ts @@ -25,7 +25,13 @@ describe('AgentChatPromptBuilder', () => { }) it('should build prompt with image asset details', () => { - const builder = new AgentChatPromptBuilder('a1').withAssetDetails('test.png', 'image/png') + const builder = new AgentChatPromptBuilder('a1').withAssetDetails( + 'test.png', + 'image/png', + undefined, + undefined, + 'image', + ) const result = builder.build() expect(result).toContain('File Name: test.png') @@ -39,6 +45,8 @@ describe('AgentChatPromptBuilder', () => { 'movie.mp4', 'video/mp4', 12.34, + undefined, + 'video', ) const result = builder.build() @@ -55,6 +63,7 @@ describe('AgentChatPromptBuilder', () => { 'application/pdf', undefined, 15, + 'pdf', ) const result = builder.build() @@ -69,7 +78,7 @@ describe('AgentChatPromptBuilder', () => { it('should build prompt with comment page for PDF asset', () => { const builder = new AgentChatPromptBuilder('a1') - .withAssetDetails('doc.pdf', 'application/pdf', undefined, 10) + .withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf') .withCommentTimestamp(3) const result = builder.build() @@ -79,7 +88,7 @@ describe('AgentChatPromptBuilder', () => { it('should build prompt for PDF continuation mode with comment page', () => { const builder = new AgentChatPromptBuilder('a1') .withContinuation(true) - .withAssetDetails('doc.pdf', 'application/pdf', undefined, 10) + .withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf') .withCommentTimestamp(4) const result = builder.build() @@ -98,7 +107,7 @@ describe('AgentChatPromptBuilder', () => { const builder = new AgentChatPromptBuilder('a1') .withExplicitMention(true) .withPathContext('src/main.ts') - .withAssetDetails('movie.mp4', 'video/mp4', 12.34) + .withAssetDetails('movie.mp4', 'video/mp4', 12.34, undefined, 'video') .withCommentTimestamp(8.2) const result = builder.build() diff --git a/packages/agent/src/workflows/agent-chat-prompt-builder.ts b/packages/agent/src/workflows/agent-chat-prompt-builder.ts index 1bf5dc5a..b6682132 100644 --- a/packages/agent/src/workflows/agent-chat-prompt-builder.ts +++ b/packages/agent/src/workflows/agent-chat-prompt-builder.ts @@ -1,19 +1,7 @@ -export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'pdf' | 'other' { - if (!mediaType) return 'other' - const lower = mediaType.toLowerCase() - if ( - lower.startsWith('image/') || - lower === 'application/x-photoshop' || - lower === 'image/vnd.adobe.photoshop' - ) { - return 'image' - } - if (lower.startsWith('video/')) { - return 'video' - } - if (lower === 'application/pdf' || lower.endsWith('.pdf')) { - return 'pdf' - } +export function getSimpleMediaType(proxyType?: string | null): 'image' | 'video' | 'pdf' | 'other' { + if (proxyType === 'image') return 'image' + if (proxyType === 'video') return 'video' + if (proxyType === 'pdf') return 'pdf' return 'other' } @@ -21,6 +9,7 @@ export class AgentChatPromptBuilder { private assetId: string private assetName?: string private mediaType?: string + private proxyType?: string private videoDuration?: number private totalPages?: number private commentTimestamp?: number @@ -64,9 +53,11 @@ export class AgentChatPromptBuilder { mediaType: string | null, duration?: number, totalPages?: number, + proxyType?: string | null, ): this { this.assetName = name this.mediaType = mediaType || undefined + this.proxyType = proxyType || undefined this.videoDuration = duration this.totalPages = totalPages return this @@ -85,10 +76,12 @@ export class AgentChatPromptBuilder { } build(): string { + const effectiveType = this.proxyType || this.mediaType + if (this.isContinuation) { let instruction = '' if (this.commentTimestamp !== undefined) { - const type = getSimpleMediaType(this.mediaType) + const type = getSimpleMediaType(effectiveType) if (type === 'pdf') { instruction += `Comment Page: ${Math.round(this.commentTimestamp)}` } else { @@ -111,8 +104,8 @@ export class AgentChatPromptBuilder { if (this.assetName) { instruction += `\nFile Name: ${this.assetName}` } - if (this.mediaType) { - const type = getSimpleMediaType(this.mediaType) + if (effectiveType) { + const type = getSimpleMediaType(effectiveType) instruction += `\nFile Type: ${type}` if (type === 'video' && this.videoDuration !== undefined) { instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds` @@ -121,7 +114,7 @@ export class AgentChatPromptBuilder { } } if (this.commentTimestamp !== undefined) { - const type = getSimpleMediaType(this.mediaType) + const type = getSimpleMediaType(effectiveType) if (type === 'pdf') { instruction += `\nComment Page: ${Math.round(this.commentTimestamp)}` } else { @@ -142,8 +135,8 @@ export class AgentChatPromptBuilder { } } - if (this.mediaType) { - const type = getSimpleMediaType(this.mediaType) + if (effectiveType) { + const type = getSimpleMediaType(effectiveType) if (type === 'image') { instruction += `\n\nIf you need to view the image data, call the 'analyze_image' tool. It does not require any parameters.` } else if (type === 'video') { diff --git a/packages/agent/src/workflows/agent-chat.test.ts b/packages/agent/src/workflows/agent-chat.test.ts index d5d05ea8..92726a65 100644 --- a/packages/agent/src/workflows/agent-chat.test.ts +++ b/packages/agent/src/workflows/agent-chat.test.ts @@ -61,6 +61,7 @@ describe('Agent Chat Workflow', () => { type: 'file', name: 'test-file.png', mediaType: 'image/png', + media: { proxyType: 'image' }, parentId: 'parent-folder-id', }) mockActivities.getCommentActivity.mockResolvedValue({ @@ -133,7 +134,7 @@ describe('Agent Chat Workflow', () => { // Verify agent instruction composition const expectedInstruction1 = new AgentChatPromptBuilder('a1') .withPathContext('Path: folder/subfolder/file.png') - .withAssetDetails('test-file.png', 'image/png', undefined) + .withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image') .withCommentTimestamp(null) .withExplicitMention(true) .build() @@ -239,7 +240,7 @@ describe('Agent Chat Workflow', () => { // Verify instruction contains not explicitly mentioned instructions const expectedInstruction3 = new AgentChatPromptBuilder('a1') .withPathContext('Path: folder/subfolder/file.png') - .withAssetDetails('test-file.png', 'image/png', undefined) + .withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image') .withCommentTimestamp(null) .withExplicitMention(false) .build() @@ -271,18 +272,21 @@ describe('Agent Chat Workflow', () => { { asset: { mediaType: 'image/png', + media: { proxyType: 'image' }, storageKey: { key: 'attachments/image1.png' }, }, }, { asset: { mediaType: 'video/mp4', // non-image, should be ignored + media: { proxyType: 'video' }, storageKey: { key: 'attachments/video.mp4' }, }, }, { asset: { mediaType: 'image/jpeg', + media: { proxyType: 'image' }, storageKey: { key: 'attachments/image2.jpg' }, }, }, @@ -305,7 +309,7 @@ describe('Agent Chat Workflow', () => { const expectedInstruction4 = new AgentChatPromptBuilder('a1') .withPathContext('Path: folder/subfolder/file.png') - .withAssetDetails('test-file.png', 'image/png', undefined) + .withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image') .withCommentTimestamp(null) .withExplicitMention(false) .build() diff --git a/packages/agent/src/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts index 7e201a17..79fa963e 100644 --- a/packages/agent/src/workflows/agent-chat.ts +++ b/packages/agent/src/workflows/agent-chat.ts @@ -74,7 +74,8 @@ export async function agentChat(task: WorkflowTask): Promise { // 5. Prepare Images (Attachments only) if (userComment.attachments) { for (const att of userComment.attachments) { - if (att.asset?.mediaType?.startsWith('image/') && att.asset?.storageKey?.key) { + const attProxyType = (att.asset?.media as PrismaJson.MediaInfo | null)?.proxyType + if (attProxyType === 'image' && att.asset?.storageKey?.key) { attachmentImageUrls.push(att.asset.storageKey.key) } } @@ -161,10 +162,11 @@ export async function agentChat(task: WorkflowTask): Promise { } } + const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType const instruction = new AgentChatPromptBuilder(asset.id) .withContinuation(!isNewChat) .withPathContext(pathContext) - .withAssetDetails(asset.name, asset.mediaType, duration, totalPages) + .withAssetDetails(asset.name, asset.mediaType, duration, totalPages, proxyType) .withCommentTimestamp(commentTimestamp) .withExplicitMention(payload.agent?.explicitMention) .withAttachedFiles(attachedFileDetailsList) diff --git a/packages/core/src/asset/asset.ts b/packages/core/src/asset/asset.ts index a5e4089f..49906182 100644 --- a/packages/core/src/asset/asset.ts +++ b/packages/core/src/asset/asset.ts @@ -2000,8 +2000,10 @@ export class AssetService { private async toPreviewInfo(asset: Asset | AssetWithIncludes): Promise { if (!asset.media) return null + const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null + let thumbnailUrl = undefined - if (asset.mediaType?.startsWith('image/') && asset.media.thumbnail?.key) { + if (proxyType === 'image' && asset.media.thumbnail?.key) { thumbnailUrl = await s3Service.presign( process.env.S3_BUCKET || 'shumai', asset.media.thumbnail.key, @@ -2024,8 +2026,6 @@ export class AssetService { ) } - const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null - return { mediaType: asset.mediaType, proxyType, @@ -2057,11 +2057,18 @@ export class AssetService { a.asset.storageKey.key, 'GET', ) + const attachmentProxyType = (a.asset.media?.proxyType || null) as + | 'image' + | 'video' + | 'audio' + | 'pdf' + | null attachments.push({ id: a.id, assetId: a.asset.id, url, mediaType: a.asset.mediaType, + proxyType: attachmentProxyType, }) } } diff --git a/packages/core/src/chat/chat.test.ts b/packages/core/src/chat/chat.test.ts index 6042cf32..9df5a515 100644 --- a/packages/core/src/chat/chat.test.ts +++ b/packages/core/src/chat/chat.test.ts @@ -181,6 +181,7 @@ describe('ChatService', () => { name: 'doc.jpg', type: 'file', mediaType: 'image/jpeg', + media: { proxyType: 'image' }, status: 'processed', projectId: project.id, parentId: childFolder.id, diff --git a/packages/core/src/chat/chat.ts b/packages/core/src/chat/chat.ts index 465c1fdf..854fc4a3 100644 --- a/packages/core/src/chat/chat.ts +++ b/packages/core/src/chat/chat.ts @@ -232,7 +232,8 @@ export class ChatService { where: { id: fileId }, include: { storageKey: true }, }) - if (file && file.mediaType?.startsWith('image/') && file.storageKey?.key) { + const isImage = (file?.media as PrismaJson.MediaInfo | null)?.proxyType === 'image' + if (file && isImage && file.storageKey?.key) { resolvedImageUrls.push(file.storageKey.key) } } diff --git a/packages/core/src/notification/notification.ts b/packages/core/src/notification/notification.ts index da257ca4..0f1012ee 100644 --- a/packages/core/src/notification/notification.ts +++ b/packages/core/src/notification/notification.ts @@ -402,6 +402,7 @@ export class NotificationService { id: a.id, name: a.name, mediaType: a.mediaType || undefined, + proxyType: (a.media as PrismaJson.MediaInfo | null)?.proxyType || undefined, preview: previewUrl || undefined, thumbnailUrl: thumbnailUrl || undefined, originalWidth: originalWidth || undefined, diff --git a/packages/core/src/upload/upload.ts b/packages/core/src/upload/upload.ts index d8c18473..e1be4dfd 100644 --- a/packages/core/src/upload/upload.ts +++ b/packages/core/src/upload/upload.ts @@ -22,6 +22,7 @@ import { ImageTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transco import { generateKeyBetween } from 'jittered-fractional-indexing' import { ulid } from 'ulid' import { sanitizeFilename } from '@shumai/core/src/utils/filename' +import { getProxyType } from '@shumai/core/src/utils/mime' export class UploadService { constructor(private readonly prismaClient: typeof prisma = prisma) {} @@ -300,16 +301,14 @@ export class UploadService { }, }) - const isVideo = asset.mediaType?.startsWith('video/') - const isImage = asset.mediaType?.startsWith('image/') - const isAudio = asset.mediaType?.startsWith('audio/') - const isPdf = - asset.mediaType === 'application/pdf' || - asset.mediaType === 'text/plain' || - asset.mediaType === 'text/csv' || - asset.name.toLowerCase().endsWith('.pdf') || - asset.name.toLowerCase().endsWith('.txt') || - asset.name.toLowerCase().endsWith('.csv') + const proxyType = + (asset.media as PrismaJson.MediaInfo | null)?.proxyType || + getProxyType(asset.mediaType, asset.name) + + const isVideo = proxyType === 'video' + const isImage = proxyType === 'image' + const isAudio = proxyType === 'audio' + const isPdf = proxyType === 'pdf' if (autofillAgent && (isVideo || isImage)) { await tx.workflowTask.create({ @@ -327,7 +326,7 @@ export class UploadService { }) } - if (!isVideo && !isImage && !isAudio && !isPdf) { + if (!proxyType) { await tx.asset.update({ where: { id: asset.id }, data: { status: AssetStatus.processed }, diff --git a/packages/core/src/utils/mime.test.ts b/packages/core/src/utils/mime.test.ts index 496eda27..f2a498f1 100644 --- a/packages/core/src/utils/mime.test.ts +++ b/packages/core/src/utils/mime.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest' -import { detectSupportedMimeType } from './mime' +import { detectSupportedMimeType, getProxyType } from './mime' describe('detectSupportedMimeType', () => { it('should detect JPEG', () => { @@ -39,3 +39,31 @@ describe('detectSupportedMimeType', () => { expect(detectSupportedMimeType(buffer)).toBeNull() }) }) + +describe('getProxyType', () => { + it('should detect image proxyType', () => { + expect(getProxyType('image/png', 'test.png')).toBe('image') + expect(getProxyType('image/jpeg', 'photo.jpg')).toBe('image') + }) + + it('should detect video proxyType', () => { + expect(getProxyType('video/mp4', 'clip.mp4')).toBe('video') + }) + + it('should detect audio proxyType', () => { + expect(getProxyType('audio/mpeg', 'song.mp3')).toBe('audio') + }) + + it('should detect pdf proxyType for pdf, csv, txt, and doc files', () => { + expect(getProxyType('application/pdf', 'doc.pdf')).toBe('pdf') + expect(getProxyType('text/plain', 'notes.txt')).toBe('pdf') + expect(getProxyType('text/csv', 'data.csv')).toBe('pdf') + expect(getProxyType('application/msword', 'letter.doc')).toBe('pdf') + expect(getProxyType(null, 'file.txt')).toBe('pdf') + }) + + it('should return null for unsupported files', () => { + expect(getProxyType('application/zip', 'archive.zip')).toBeNull() + expect(getProxyType(null, 'unknown.bin')).toBeNull() + }) +}) diff --git a/packages/core/src/utils/mime.ts b/packages/core/src/utils/mime.ts index ca4e4b7a..9310499d 100644 --- a/packages/core/src/utils/mime.ts +++ b/packages/core/src/utils/mime.ts @@ -1,3 +1,40 @@ +export type ProxyType = 'image' | 'video' | 'audio' | 'pdf' + +export function getProxyType( + mediaType?: string | null, + filename?: string | null, +): ProxyType | null { + const lowerMediaType = mediaType?.toLowerCase() || '' + const lowerFilename = filename?.toLowerCase() || '' + + if (lowerMediaType.startsWith('image/')) return 'image' + if (lowerMediaType.startsWith('video/')) return 'video' + if (lowerMediaType.startsWith('audio/')) return 'audio' + + if ( + lowerMediaType === 'application/pdf' || + lowerMediaType === 'text/plain' || + lowerMediaType === 'text/csv' || + lowerMediaType.includes('msword') || + lowerMediaType.includes('officedocument') || + lowerMediaType.includes('vnd.ms-') || + lowerMediaType.startsWith('text/') || + lowerFilename.endsWith('.pdf') || + lowerFilename.endsWith('.txt') || + lowerFilename.endsWith('.csv') || + lowerFilename.endsWith('.doc') || + lowerFilename.endsWith('.docx') || + lowerFilename.endsWith('.xls') || + lowerFilename.endsWith('.xlsx') || + lowerFilename.endsWith('.ppt') || + lowerFilename.endsWith('.pptx') + ) { + return 'pdf' + } + + return null +} + const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] export function detectSupportedMimeType(buffer: Uint8Array): string | null { diff --git a/packages/dtos/src/asset.ts b/packages/dtos/src/asset.ts index faa2ff07..b9838b6c 100644 --- a/packages/dtos/src/asset.ts +++ b/packages/dtos/src/asset.ts @@ -191,6 +191,7 @@ export const attachmentInfoSchema = z.object({ assetId: z.string(), url: z.string(), mediaType: z.string().nullable(), + proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(), }) export type AttachmentInfo = z.infer diff --git a/packages/dtos/src/notification.ts b/packages/dtos/src/notification.ts index 024c6646..ae495503 100644 --- a/packages/dtos/src/notification.ts +++ b/packages/dtos/src/notification.ts @@ -19,6 +19,7 @@ export type EntityInfo = z.infer export const assetPreviewSchema = z.object({ id: z.string().optional(), mediaType: z.string().optional(), + proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(), name: z.string().optional(), preview: z.string().optional(), thumbnailUrl: z.string().optional(), diff --git a/packages/e2e/workflow/agent-autofill.test.ts b/packages/e2e/workflow/agent-autofill.test.ts index 91e78390..316019fa 100644 --- a/packages/e2e/workflow/agent-autofill.test.ts +++ b/packages/e2e/workflow/agent-autofill.test.ts @@ -230,6 +230,7 @@ describe.each(['local', 'temporal'] as const)( type: 'file', status: 'uploaded', mediaType: 'image/png', + media: { proxyType: 'image' }, projectId: project.id, storageKeyId: storageKey.id, }, diff --git a/packages/transcode/src/activities/transcode.test.ts b/packages/transcode/src/activities/transcode.test.ts index 04b88cb1..ab27db51 100644 --- a/packages/transcode/src/activities/transcode.test.ts +++ b/packages/transcode/src/activities/transcode.test.ts @@ -117,6 +117,7 @@ describe('Transcode Activities', () => { const result = await getMediaInfoActivity({ assetId: asset.id, filePath: '/tmp/v.mp4', + proxyType: 'video', mediaType: 'video/mp4', }) @@ -153,6 +154,7 @@ describe('Transcode Activities', () => { const result = await getMediaInfoActivity({ assetId: asset.id, filePath: '/tmp/i.png', + proxyType: 'image', mediaType: 'image/png', }) @@ -188,6 +190,7 @@ describe('Transcode Activities', () => { const result = await getMediaInfoActivity({ assetId: asset.id, filePath: '/tmp/a.mp3', + proxyType: 'audio', mediaType: 'audio/mpeg', }) @@ -221,6 +224,7 @@ describe('Transcode Activities', () => { await getMediaInfoActivity({ assetId: asset.id, filePath: '/tmp/d.pdf', + proxyType: 'pdf', mediaType: 'application/pdf', }) diff --git a/packages/transcode/src/activities/transcode.ts b/packages/transcode/src/activities/transcode.ts index 5223df58..ceb366b6 100644 --- a/packages/transcode/src/activities/transcode.ts +++ b/packages/transcode/src/activities/transcode.ts @@ -3,6 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3' import { transcodeService } from '@shumai/core/src/transcode/transcode' import { metadataService } from '@shumai/core/src/metadata/metadata' import { stemFromKey } from '@shumai/core/src/utils/filename' +import { getProxyType } from '@shumai/core/src/utils/mime' import { ApplicationFailure } from '@temporalio/activity' import * as path from 'path' import * as fs from 'fs' @@ -24,7 +25,12 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?: const record = err as Record return { code: typeof record.code === 'string' ? record.code : undefined, - message: typeof record.message === 'string' ? record.message : String(err), + message: + typeof record.message === 'string' + ? record.message + : err instanceof Error + ? err.message + : String(err), name: typeof record.name === 'string' ? record.name : undefined, } } @@ -36,18 +42,16 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?: export async function getMediaInfoActivity(params: { filePath: string assetId: string - mediaType: string + proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null + mediaType?: string }): Promise { try { - const isVideo = params.mediaType.startsWith('video/') - const isImage = params.mediaType.startsWith('image/') - const isAudio = params.mediaType.startsWith('audio/') - const isDocument = - params.mediaType.startsWith('text/') || - params.mediaType === 'application/pdf' || - params.mediaType.includes('msword') || - params.mediaType.includes('officedocument') || - params.mediaType.includes('vnd.ms-') + const proxyType = + params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined + const isVideo = proxyType === 'video' + const isImage = proxyType === 'image' + const isAudio = proxyType === 'audio' + const isPdf = proxyType === 'pdf' const fileType = isVideo ? 'video' @@ -55,25 +59,10 @@ export async function getMediaInfoActivity(params: { ? 'audio' : isImage ? 'image' - : isDocument + : isPdf ? 'document' : 'file' - const isPdf = - params.mediaType === 'application/pdf' || - params.mediaType === 'text/plain' || - params.mediaType === 'text/csv' - - const proxyType: 'image' | 'video' | 'audio' | 'pdf' | undefined = isVideo - ? 'video' - : isAudio - ? 'audio' - : isImage - ? 'image' - : isPdf || isDocument - ? 'pdf' - : undefined - const mediaInfo: PrismaJson.MediaInfo = { duration: 0, filesize: 0, @@ -84,7 +73,7 @@ export async function getMediaInfoActivity(params: { videoPreview: { width: 0, height: 0 }, finishedAt: new Date().toISOString(), metadata: null, - mimeType: params.mediaType, + mimeType: params.mediaType || '', original: { key: '', // Will be filled by caller or updated later downloadUrl: '', @@ -182,7 +171,7 @@ export async function getMediaInfoActivity(params: { { key: 'resolution_width', value: info.originalWidth }, { key: 'resolution_height', value: info.originalHeight }, ) - } else if (isPdf || isDocument) { + } else if (isPdf) { const info = await transcodeService.getPdfInfo(params.filePath) mediaInfo.frames = info.totalFrames mediaInfo.metadata = { diff --git a/packages/transcode/src/workflows/render-pdf-pages.ts b/packages/transcode/src/workflows/render-pdf-pages.ts index 41e3fe43..fbad3c4d 100644 --- a/packages/transcode/src/workflows/render-pdf-pages.ts +++ b/packages/transcode/src/workflows/render-pdf-pages.ts @@ -18,9 +18,10 @@ export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId) const { renderPdfPagesActivity } = getActivities() + const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key const pages = await executeActivity(workerQueue, renderPdfPagesActivity, { - assetKey: key, + assetKey: pdfKey, assetId: asset.id, start: payload.pdfPages.start, end: payload.pdfPages.end, diff --git a/packages/transcode/src/workflows/transcode-image.ts b/packages/transcode/src/workflows/transcode-image.ts index 2166004d..8639ac3f 100644 --- a/packages/transcode/src/workflows/transcode-image.ts +++ b/packages/transcode/src/workflows/transcode-image.ts @@ -43,6 +43,7 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, { filePath, assetId: asset.id, + proxyType: 'image', mediaType: asset.mediaType || '', }) diff --git a/packages/transcode/src/workflows/transcode-pdf.ts b/packages/transcode/src/workflows/transcode-pdf.ts index d89d872d..071fe2b1 100644 --- a/packages/transcode/src/workflows/transcode-pdf.ts +++ b/packages/transcode/src/workflows/transcode-pdf.ts @@ -53,6 +53,7 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise { const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, { filePath: currentFilePath, assetId: asset.id, + proxyType: 'pdf', mediaType: asset.mediaType || '', }) diff --git a/packages/transcode/src/workflows/transcode-video.ts b/packages/transcode/src/workflows/transcode-video.ts index 530ea9a5..1afa9c12 100644 --- a/packages/transcode/src/workflows/transcode-video.ts +++ b/packages/transcode/src/workflows/transcode-video.ts @@ -43,9 +43,11 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise tmpDir = download.tmpDir const spec = task.payload?.transcode || {} + const proxyType = asset.mediaType?.startsWith('audio/') ? 'audio' : 'video' const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, { filePath, assetId: asset.id, + proxyType, mediaType: asset.mediaType || '', }) diff --git a/packages/webui/components/breadcrumb-nav.tsx b/packages/webui/components/breadcrumb-nav.tsx index 04049de9..d68b6941 100644 --- a/packages/webui/components/breadcrumb-nav.tsx +++ b/packages/webui/components/breadcrumb-nav.tsx @@ -51,6 +51,7 @@ interface BreadcrumbNavProps { type: 'file' | 'folder' version?: number mediaType?: string | null + proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null } isRootFolder: boolean displayStyle?: 'card' | 'list' @@ -158,7 +159,7 @@ export function BreadcrumbNav({ } } - const isAudio = currentAsset.mediaType?.startsWith('audio/') + const isAudio = currentAsset.proxyType === 'audio' const hasVideoTranscodes = !isAudio && downloadInfo?.videoTranscodes && diff --git a/packages/webui/components/chat/message-card.tsx b/packages/webui/components/chat/message-card.tsx index 9bd9f4d4..00e81cd1 100644 --- a/packages/webui/components/chat/message-card.tsx +++ b/packages/webui/components/chat/message-card.tsx @@ -346,7 +346,7 @@ export const MessageCard: React.FC = ({ {message.attachments && message.attachments.length > 0 && (
{message.attachments.map((att) => { - const isImage = att.mediaType?.startsWith('image/') + const isImage = att.proxyType === 'image' const name = att.url?.split('/').pop()?.split('?')[0] || 'file' return (
{ describe('kindOf', () => { it('classifies image media types', () => { - expect(kindOf('image/png')).toBe('image') - expect(kindOf('image/webp')).toBe('image') + expect(kindOf('image')).toBe('image') + expect(kindOf({ proxyType: 'image' })).toBe('image') }) it('classifies video media types', () => { - expect(kindOf('video/mp4')).toBe('video') - expect(kindOf('video/quicktime')).toBe('video') + expect(kindOf('video')).toBe('video') + expect(kindOf({ proxyType: 'video' })).toBe('video') }) it('returns unsupported for missing or unknown types', () => { expect(kindOf(null)).toBe('unsupported') expect(kindOf(undefined)).toBe('unsupported') - expect(kindOf('application/pdf')).toBe('unsupported') + expect(kindOf('pdf')).toBe('unsupported') }) }) diff --git a/packages/webui/components/compare/compare-utils.ts b/packages/webui/components/compare/compare-utils.ts index dace8b6a..48da25ad 100644 --- a/packages/webui/components/compare/compare-utils.ts +++ b/packages/webui/components/compare/compare-utils.ts @@ -1,10 +1,11 @@ import type { PaneKind } from './types' /** Classify an asset's media type into a compare pane kind. */ -export function kindOf(mediaType: string | null | undefined): PaneKind { - if (!mediaType) return 'unsupported' - if (mediaType.startsWith('image/')) return 'image' - if (mediaType.startsWith('video/')) return 'video' +export function kindOf(item: { proxyType?: string | null } | string | null | undefined): PaneKind { + if (!item) return 'unsupported' + const proxyType = typeof item === 'object' ? item.proxyType : item + if (proxyType === 'image') return 'image' + if (proxyType === 'video') return 'video' return 'unsupported' } diff --git a/packages/webui/components/file-browser/file-list-item.tsx b/packages/webui/components/file-browser/file-list-item.tsx index 1704b1f6..0e33ce11 100644 --- a/packages/webui/components/file-browser/file-list-item.tsx +++ b/packages/webui/components/file-browser/file-list-item.tsx @@ -214,8 +214,7 @@ export function FileListItem({ alt="" className="w-full h-full object-cover" /> - ) : displayItem.preview?.mediaType?.startsWith('audio/') || - displayItem.mediaType?.startsWith('audio/') ? ( + ) : displayItem.preview?.proxyType === 'audio' || displayItem.proxyType === 'audio' ? ( ) : displayItem.type === 'folder' ? ( diff --git a/packages/webui/components/file-viewer-left-sidebar.tsx b/packages/webui/components/file-viewer-left-sidebar.tsx index 098648cf..cd1f5789 100644 --- a/packages/webui/components/file-viewer-left-sidebar.tsx +++ b/packages/webui/components/file-viewer-left-sidebar.tsx @@ -22,8 +22,7 @@ export const CarouselFilePreview = ({ item }: { item: AssetInfo }) => { ) } - const isAudio = - item.preview?.mediaType?.startsWith('audio/') || item.mediaType?.startsWith('audio/') + const isAudio = item.preview?.proxyType === 'audio' || item.proxyType === 'audio' if (isAudio) { return (
diff --git a/packages/webui/components/file-viewer-right-sidebar.tsx b/packages/webui/components/file-viewer-right-sidebar.tsx index c9cfead1..ceb58e04 100644 --- a/packages/webui/components/file-viewer-right-sidebar.tsx +++ b/packages/webui/components/file-viewer-right-sidebar.tsx @@ -315,7 +315,7 @@ export function FileViewerRightSidebar({ return (
{/* Lightbox */} - {viewingAttachment && viewingAttachment.mediaType?.startsWith('image/') && ( + {viewingAttachment && viewingAttachment.proxyType === 'image' && (
setViewingAttachment(null)} diff --git a/packages/webui/components/notification-card.tsx b/packages/webui/components/notification-card.tsx index b17f0405..8c4f90e1 100644 --- a/packages/webui/components/notification-card.tsx +++ b/packages/webui/components/notification-card.tsx @@ -92,14 +92,13 @@ export const NotificationCard = ({ notification }: NotificationCardProps) => { })() const isVideoWithSprite = - asset?.mediaType?.startsWith('video/') && + asset?.proxyType === 'video' && asset.preview && asset.thumbnailUrl && asset.originalWidth && asset.originalHeight - const hasPreview = - asset?.preview && (asset.mediaType?.startsWith('image') || asset.mediaType?.startsWith('video')) + const hasPreview = asset?.preview && (asset.proxyType === 'image' || asset.proxyType === 'video') return (
diff --git a/packages/webui/components/public-share-manager.tsx b/packages/webui/components/public-share-manager.tsx index 3c35a603..037ab49a 100644 --- a/packages/webui/components/public-share-manager.tsx +++ b/packages/webui/components/public-share-manager.tsx @@ -647,10 +647,9 @@ export function PublicShareManager({ mediaControllerRef.current?.pause() }} selectedCommentId={selectedCommentId} - hideAnnotationControl={(isCompareMode - ? compareActiveAsset - : currentSelectedItem - )?.mediaType?.startsWith('audio/')} + hideAnnotationControl={ + (isCompareMode ? compareActiveAsset : currentSelectedItem)?.proxyType === 'audio' + } />
diff --git a/packages/webui/components/viewers/video/compare-video-pane.tsx b/packages/webui/components/viewers/video/compare-video-pane.tsx index 1c49d838..a5376799 100644 --- a/packages/webui/components/viewers/video/compare-video-pane.tsx +++ b/packages/webui/components/viewers/video/compare-video-pane.tsx @@ -118,7 +118,7 @@ export const CompareVideoPane = forwardRef
- {file.mediaType?.startsWith('audio/') ? ( + {isAudio ? (
= ({ // fullscreen). Compare keeps it in-flow. Colors stay theme-aware either way. const overlay = state.isFullScreen && floatOverlayInFullScreen - const isAudio = data.mediaType?.startsWith('audio/') + const isAudio = data.proxyType === 'audio' const previewResolutions = resolutions.filter((r) => !r.isRaw) const startTimecode = data.media?.metadata?.startTimecode diff --git a/packages/webui/components/viewers/video/video-viewer.tsx b/packages/webui/components/viewers/video/video-viewer.tsx index 74727c35..ddc88e7f 100644 --- a/packages/webui/components/viewers/video/video-viewer.tsx +++ b/packages/webui/components/viewers/video/video-viewer.tsx @@ -194,7 +194,7 @@ const VideoViewer = React.forwardRef( const dbTotalFrames = metadata?.totalFrames || 0 const containerDuration = metadata?.duration || 0 - const isAudio = data.mediaType?.startsWith('audio/') + const isAudio = data.proxyType === 'audio' const totalFrames = resolveTotalFrames({ dbTotalFrames, containerDuration, frameRate }) const { currentFrame, seekToFrame } = useFramePlayer(videoRef, frameRate, totalFrames, isAudio) @@ -626,7 +626,7 @@ const VideoViewer = React.forwardRef( {/* Hidden VideoJS container */}
- {data.mediaType?.startsWith('audio/') ? ( + {isAudio ? (
)}