feat(asset): replace mediaType/mimeType logic with proxyType

This commit is contained in:
yiling
2026-07-20 10:22:36 +08:00
parent c45a1d9ee9
commit f158d04686
40 changed files with 219 additions and 119 deletions
+3 -2
View File
@@ -88,8 +88,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
outputTokens: 0,
}
const isImage = asset.mediaType.startsWith('image/')
const isVideo = asset.mediaType.startsWith('video/')
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
if (!isImage && !isVideo) {
throw ApplicationFailure.create({
@@ -409,6 +409,7 @@ describe('DatabaseSessionStorage', () => {
name: 'test.png',
type: 'file',
mediaType: 'image/png',
media: { proxyType: 'image' },
projectId: project.id,
status: 'uploaded',
},
@@ -461,6 +462,7 @@ describe('DatabaseSessionStorage', () => {
name: 'test.mp4',
type: 'file',
mediaType: 'video/mp4',
media: { proxyType: 'video' },
projectId: project.id,
status: 'uploaded',
},
+8 -9
View File
@@ -235,15 +235,14 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
const asset = await prisma.asset.findUnique({
where: { id: assetId },
})
if (asset?.mediaType) {
const type = getSimpleMediaType(asset.mediaType)
if (type === 'image') {
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
} else if (type === 'video') {
mediaTools.push(createScreenshotTool(assetId, userCommentId))
} else if (type === 'pdf') {
mediaTools.push(createReadPdfPagesTool(assetId, userCommentId))
}
const proxyType = (asset?.media as PrismaJson.MediaInfo | null)?.proxyType
const type = getSimpleMediaType(proxyType)
if (type === 'image') {
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
} else if (type === 'video') {
mediaTools.push(createScreenshotTool(assetId, userCommentId))
} else if (type === 'pdf') {
mediaTools.push(createReadPdfPagesTool(assetId, userCommentId))
}
}
@@ -77,6 +77,14 @@ export function createReadPdfPagesTool(
}
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
if (proxyType && proxyType !== 'pdf') {
return {
content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }],
details: {},
}
}
// Fetch trigger comment details for comment page and annotations
let commentTimestamp: number | null = null
let annotations: unknown = null
@@ -64,6 +64,7 @@ describe('Agent Autofill Workflow', () => {
storageKey: { key: 'asset-key' },
project: { id: 'p1', teamId: 't1' },
mediaType: 'image/png',
media: { proxyType: 'image' },
})
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
filePath: '/tmp/test.png',
@@ -290,6 +291,7 @@ describe('Agent Autofill Workflow', () => {
storageKey: { key: 'asset-key' },
project: { id: 'p1', teamId: 't1' },
mediaType: 'audio/mp3',
media: { proxyType: 'audio' },
})
const task = await prisma.workflowTask.create({
@@ -64,8 +64,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
const projectId = asset.project.id
// 2. Prepare Data (Images)
const isImage = asset.mediaType?.startsWith('image/') || false
const isVideo = asset.mediaType?.startsWith('video/') || false
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const isImage = proxyType === 'image'
const isVideo = proxyType === 'video'
if (!isImage && !isVideo) {
throw ApplicationFailure.create({
@@ -25,7 +25,13 @@ describe('AgentChatPromptBuilder', () => {
})
it('should build prompt with image asset details', () => {
const builder = new AgentChatPromptBuilder('a1').withAssetDetails('test.png', 'image/png')
const builder = new AgentChatPromptBuilder('a1').withAssetDetails(
'test.png',
'image/png',
undefined,
undefined,
'image',
)
const result = builder.build()
expect(result).toContain('File Name: test.png')
@@ -39,6 +45,8 @@ describe('AgentChatPromptBuilder', () => {
'movie.mp4',
'video/mp4',
12.34,
undefined,
'video',
)
const result = builder.build()
@@ -55,6 +63,7 @@ describe('AgentChatPromptBuilder', () => {
'application/pdf',
undefined,
15,
'pdf',
)
const result = builder.build()
@@ -69,7 +78,7 @@ describe('AgentChatPromptBuilder', () => {
it('should build prompt with comment page for PDF asset', () => {
const builder = new AgentChatPromptBuilder('a1')
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10)
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
.withCommentTimestamp(3)
const result = builder.build()
@@ -79,7 +88,7 @@ describe('AgentChatPromptBuilder', () => {
it('should build prompt for PDF continuation mode with comment page', () => {
const builder = new AgentChatPromptBuilder('a1')
.withContinuation(true)
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10)
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
.withCommentTimestamp(4)
const result = builder.build()
@@ -98,7 +107,7 @@ describe('AgentChatPromptBuilder', () => {
const builder = new AgentChatPromptBuilder('a1')
.withExplicitMention(true)
.withPathContext('src/main.ts')
.withAssetDetails('movie.mp4', 'video/mp4', 12.34)
.withAssetDetails('movie.mp4', 'video/mp4', 12.34, undefined, 'video')
.withCommentTimestamp(8.2)
const result = builder.build()
@@ -1,19 +1,7 @@
export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'pdf' | 'other' {
if (!mediaType) return 'other'
const lower = mediaType.toLowerCase()
if (
lower.startsWith('image/') ||
lower === 'application/x-photoshop' ||
lower === 'image/vnd.adobe.photoshop'
) {
return 'image'
}
if (lower.startsWith('video/')) {
return 'video'
}
if (lower === 'application/pdf' || lower.endsWith('.pdf')) {
return 'pdf'
}
export function getSimpleMediaType(proxyType?: string | null): 'image' | 'video' | 'pdf' | 'other' {
if (proxyType === 'image') return 'image'
if (proxyType === 'video') return 'video'
if (proxyType === 'pdf') return 'pdf'
return 'other'
}
@@ -21,6 +9,7 @@ export class AgentChatPromptBuilder {
private assetId: string
private assetName?: string
private mediaType?: string
private proxyType?: string
private videoDuration?: number
private totalPages?: number
private commentTimestamp?: number
@@ -64,9 +53,11 @@ export class AgentChatPromptBuilder {
mediaType: string | null,
duration?: number,
totalPages?: number,
proxyType?: string | null,
): this {
this.assetName = name
this.mediaType = mediaType || undefined
this.proxyType = proxyType || undefined
this.videoDuration = duration
this.totalPages = totalPages
return this
@@ -85,10 +76,12 @@ export class AgentChatPromptBuilder {
}
build(): string {
const effectiveType = this.proxyType || this.mediaType
if (this.isContinuation) {
let instruction = ''
if (this.commentTimestamp !== undefined) {
const type = getSimpleMediaType(this.mediaType)
const type = getSimpleMediaType(effectiveType)
if (type === 'pdf') {
instruction += `Comment Page: ${Math.round(this.commentTimestamp)}`
} else {
@@ -111,8 +104,8 @@ export class AgentChatPromptBuilder {
if (this.assetName) {
instruction += `\nFile Name: ${this.assetName}`
}
if (this.mediaType) {
const type = getSimpleMediaType(this.mediaType)
if (effectiveType) {
const type = getSimpleMediaType(effectiveType)
instruction += `\nFile Type: ${type}`
if (type === 'video' && this.videoDuration !== undefined) {
instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds`
@@ -121,7 +114,7 @@ export class AgentChatPromptBuilder {
}
}
if (this.commentTimestamp !== undefined) {
const type = getSimpleMediaType(this.mediaType)
const type = getSimpleMediaType(effectiveType)
if (type === 'pdf') {
instruction += `\nComment Page: ${Math.round(this.commentTimestamp)}`
} else {
@@ -142,8 +135,8 @@ export class AgentChatPromptBuilder {
}
}
if (this.mediaType) {
const type = getSimpleMediaType(this.mediaType)
if (effectiveType) {
const type = getSimpleMediaType(effectiveType)
if (type === 'image') {
instruction += `\n\nIf you need to view the image data, call the 'analyze_image' tool. It does not require any parameters.`
} else if (type === 'video') {
@@ -61,6 +61,7 @@ describe('Agent Chat Workflow', () => {
type: 'file',
name: 'test-file.png',
mediaType: 'image/png',
media: { proxyType: 'image' },
parentId: 'parent-folder-id',
})
mockActivities.getCommentActivity.mockResolvedValue({
@@ -133,7 +134,7 @@ describe('Agent Chat Workflow', () => {
// Verify agent instruction composition
const expectedInstruction1 = new AgentChatPromptBuilder('a1')
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('test-file.png', 'image/png', undefined)
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
.withCommentTimestamp(null)
.withExplicitMention(true)
.build()
@@ -239,7 +240,7 @@ describe('Agent Chat Workflow', () => {
// Verify instruction contains not explicitly mentioned instructions
const expectedInstruction3 = new AgentChatPromptBuilder('a1')
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('test-file.png', 'image/png', undefined)
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
.withCommentTimestamp(null)
.withExplicitMention(false)
.build()
@@ -271,18 +272,21 @@ describe('Agent Chat Workflow', () => {
{
asset: {
mediaType: 'image/png',
media: { proxyType: 'image' },
storageKey: { key: 'attachments/image1.png' },
},
},
{
asset: {
mediaType: 'video/mp4', // non-image, should be ignored
media: { proxyType: 'video' },
storageKey: { key: 'attachments/video.mp4' },
},
},
{
asset: {
mediaType: 'image/jpeg',
media: { proxyType: 'image' },
storageKey: { key: 'attachments/image2.jpg' },
},
},
@@ -305,7 +309,7 @@ describe('Agent Chat Workflow', () => {
const expectedInstruction4 = new AgentChatPromptBuilder('a1')
.withPathContext('Path: folder/subfolder/file.png')
.withAssetDetails('test-file.png', 'image/png', undefined)
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
.withCommentTimestamp(null)
.withExplicitMention(false)
.build()
+4 -2
View File
@@ -74,7 +74,8 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
// 5. Prepare Images (Attachments only)
if (userComment.attachments) {
for (const att of userComment.attachments) {
if (att.asset?.mediaType?.startsWith('image/') && att.asset?.storageKey?.key) {
const attProxyType = (att.asset?.media as PrismaJson.MediaInfo | null)?.proxyType
if (attProxyType === 'image' && att.asset?.storageKey?.key) {
attachmentImageUrls.push(att.asset.storageKey.key)
}
}
@@ -161,10 +162,11 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
}
}
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
const instruction = new AgentChatPromptBuilder(asset.id)
.withContinuation(!isNewChat)
.withPathContext(pathContext)
.withAssetDetails(asset.name, asset.mediaType, duration, totalPages)
.withAssetDetails(asset.name, asset.mediaType, duration, totalPages, proxyType)
.withCommentTimestamp(commentTimestamp)
.withExplicitMention(payload.agent?.explicitMention)
.withAttachedFiles(attachedFileDetailsList)
+10 -3
View File
@@ -2000,8 +2000,10 @@ export class AssetService {
private async toPreviewInfo(asset: Asset | AssetWithIncludes): Promise<PreviewInfo | null> {
if (!asset.media) return null
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
let thumbnailUrl = undefined
if (asset.mediaType?.startsWith('image/') && asset.media.thumbnail?.key) {
if (proxyType === 'image' && asset.media.thumbnail?.key) {
thumbnailUrl = await s3Service.presign(
process.env.S3_BUCKET || 'shumai',
asset.media.thumbnail.key,
@@ -2024,8 +2026,6 @@ export class AssetService {
)
}
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
return {
mediaType: asset.mediaType,
proxyType,
@@ -2057,11 +2057,18 @@ export class AssetService {
a.asset.storageKey.key,
'GET',
)
const attachmentProxyType = (a.asset.media?.proxyType || null) as
| 'image'
| 'video'
| 'audio'
| 'pdf'
| null
attachments.push({
id: a.id,
assetId: a.asset.id,
url,
mediaType: a.asset.mediaType,
proxyType: attachmentProxyType,
})
}
}
+1
View File
@@ -181,6 +181,7 @@ describe('ChatService', () => {
name: 'doc.jpg',
type: 'file',
mediaType: 'image/jpeg',
media: { proxyType: 'image' },
status: 'processed',
projectId: project.id,
parentId: childFolder.id,
+2 -1
View File
@@ -232,7 +232,8 @@ export class ChatService {
where: { id: fileId },
include: { storageKey: true },
})
if (file && file.mediaType?.startsWith('image/') && file.storageKey?.key) {
const isImage = (file?.media as PrismaJson.MediaInfo | null)?.proxyType === 'image'
if (file && isImage && file.storageKey?.key) {
resolvedImageUrls.push(file.storageKey.key)
}
}
@@ -402,6 +402,7 @@ export class NotificationService {
id: a.id,
name: a.name,
mediaType: a.mediaType || undefined,
proxyType: (a.media as PrismaJson.MediaInfo | null)?.proxyType || undefined,
preview: previewUrl || undefined,
thumbnailUrl: thumbnailUrl || undefined,
originalWidth: originalWidth || undefined,
+10 -11
View File
@@ -22,6 +22,7 @@ import { ImageTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transco
import { generateKeyBetween } from 'jittered-fractional-indexing'
import { ulid } from 'ulid'
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
import { getProxyType } from '@shumai/core/src/utils/mime'
export class UploadService {
constructor(private readonly prismaClient: typeof prisma = prisma) {}
@@ -300,16 +301,14 @@ export class UploadService {
},
})
const isVideo = asset.mediaType?.startsWith('video/')
const isImage = asset.mediaType?.startsWith('image/')
const isAudio = asset.mediaType?.startsWith('audio/')
const isPdf =
asset.mediaType === 'application/pdf' ||
asset.mediaType === 'text/plain' ||
asset.mediaType === 'text/csv' ||
asset.name.toLowerCase().endsWith('.pdf') ||
asset.name.toLowerCase().endsWith('.txt') ||
asset.name.toLowerCase().endsWith('.csv')
const proxyType =
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
getProxyType(asset.mediaType, asset.name)
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
if (autofillAgent && (isVideo || isImage)) {
await tx.workflowTask.create({
@@ -327,7 +326,7 @@ export class UploadService {
})
}
if (!isVideo && !isImage && !isAudio && !isPdf) {
if (!proxyType) {
await tx.asset.update({
where: { id: asset.id },
data: { status: AssetStatus.processed },
+29 -1
View File
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest'
import { detectSupportedMimeType } from './mime'
import { detectSupportedMimeType, getProxyType } from './mime'
describe('detectSupportedMimeType', () => {
it('should detect JPEG', () => {
@@ -39,3 +39,31 @@ describe('detectSupportedMimeType', () => {
expect(detectSupportedMimeType(buffer)).toBeNull()
})
})
describe('getProxyType', () => {
it('should detect image proxyType', () => {
expect(getProxyType('image/png', 'test.png')).toBe('image')
expect(getProxyType('image/jpeg', 'photo.jpg')).toBe('image')
})
it('should detect video proxyType', () => {
expect(getProxyType('video/mp4', 'clip.mp4')).toBe('video')
})
it('should detect audio proxyType', () => {
expect(getProxyType('audio/mpeg', 'song.mp3')).toBe('audio')
})
it('should detect pdf proxyType for pdf, csv, txt, and doc files', () => {
expect(getProxyType('application/pdf', 'doc.pdf')).toBe('pdf')
expect(getProxyType('text/plain', 'notes.txt')).toBe('pdf')
expect(getProxyType('text/csv', 'data.csv')).toBe('pdf')
expect(getProxyType('application/msword', 'letter.doc')).toBe('pdf')
expect(getProxyType(null, 'file.txt')).toBe('pdf')
})
it('should return null for unsupported files', () => {
expect(getProxyType('application/zip', 'archive.zip')).toBeNull()
expect(getProxyType(null, 'unknown.bin')).toBeNull()
})
})
+37
View File
@@ -1,3 +1,40 @@
export type ProxyType = 'image' | 'video' | 'audio' | 'pdf'
export function getProxyType(
mediaType?: string | null,
filename?: string | null,
): ProxyType | null {
const lowerMediaType = mediaType?.toLowerCase() || ''
const lowerFilename = filename?.toLowerCase() || ''
if (lowerMediaType.startsWith('image/')) return 'image'
if (lowerMediaType.startsWith('video/')) return 'video'
if (lowerMediaType.startsWith('audio/')) return 'audio'
if (
lowerMediaType === 'application/pdf' ||
lowerMediaType === 'text/plain' ||
lowerMediaType === 'text/csv' ||
lowerMediaType.includes('msword') ||
lowerMediaType.includes('officedocument') ||
lowerMediaType.includes('vnd.ms-') ||
lowerMediaType.startsWith('text/') ||
lowerFilename.endsWith('.pdf') ||
lowerFilename.endsWith('.txt') ||
lowerFilename.endsWith('.csv') ||
lowerFilename.endsWith('.doc') ||
lowerFilename.endsWith('.docx') ||
lowerFilename.endsWith('.xls') ||
lowerFilename.endsWith('.xlsx') ||
lowerFilename.endsWith('.ppt') ||
lowerFilename.endsWith('.pptx')
) {
return 'pdf'
}
return null
}
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
export function detectSupportedMimeType(buffer: Uint8Array): string | null {
+1
View File
@@ -191,6 +191,7 @@ export const attachmentInfoSchema = z.object({
assetId: z.string(),
url: z.string(),
mediaType: z.string().nullable(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
})
export type AttachmentInfo = z.infer<typeof attachmentInfoSchema>
+1
View File
@@ -19,6 +19,7 @@ export type EntityInfo = z.infer<typeof entityInfoSchema>
export const assetPreviewSchema = z.object({
id: z.string().optional(),
mediaType: z.string().optional(),
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
name: z.string().optional(),
preview: z.string().optional(),
thumbnailUrl: z.string().optional(),
@@ -230,6 +230,7 @@ describe.each(['local', 'temporal'] as const)(
type: 'file',
status: 'uploaded',
mediaType: 'image/png',
media: { proxyType: 'image' },
projectId: project.id,
storageKeyId: storageKey.id,
},
@@ -117,6 +117,7 @@ describe('Transcode Activities', () => {
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/v.mp4',
proxyType: 'video',
mediaType: 'video/mp4',
})
@@ -153,6 +154,7 @@ describe('Transcode Activities', () => {
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/i.png',
proxyType: 'image',
mediaType: 'image/png',
})
@@ -188,6 +190,7 @@ describe('Transcode Activities', () => {
const result = await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/a.mp3',
proxyType: 'audio',
mediaType: 'audio/mpeg',
})
@@ -221,6 +224,7 @@ describe('Transcode Activities', () => {
await getMediaInfoActivity({
assetId: asset.id,
filePath: '/tmp/d.pdf',
proxyType: 'pdf',
mediaType: 'application/pdf',
})
+18 -29
View File
@@ -3,6 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
import { transcodeService } from '@shumai/core/src/transcode/transcode'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { stemFromKey } from '@shumai/core/src/utils/filename'
import { getProxyType } from '@shumai/core/src/utils/mime'
import { ApplicationFailure } from '@temporalio/activity'
import * as path from 'path'
import * as fs from 'fs'
@@ -24,7 +25,12 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
const record = err as Record<string, unknown>
return {
code: typeof record.code === 'string' ? record.code : undefined,
message: typeof record.message === 'string' ? record.message : String(err),
message:
typeof record.message === 'string'
? record.message
: err instanceof Error
? err.message
: String(err),
name: typeof record.name === 'string' ? record.name : undefined,
}
}
@@ -36,18 +42,16 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
export async function getMediaInfoActivity(params: {
filePath: string
assetId: string
mediaType: string
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
mediaType?: string
}): Promise<PrismaJson.MediaInfo> {
try {
const isVideo = params.mediaType.startsWith('video/')
const isImage = params.mediaType.startsWith('image/')
const isAudio = params.mediaType.startsWith('audio/')
const isDocument =
params.mediaType.startsWith('text/') ||
params.mediaType === 'application/pdf' ||
params.mediaType.includes('msword') ||
params.mediaType.includes('officedocument') ||
params.mediaType.includes('vnd.ms-')
const proxyType =
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
const isVideo = proxyType === 'video'
const isImage = proxyType === 'image'
const isAudio = proxyType === 'audio'
const isPdf = proxyType === 'pdf'
const fileType = isVideo
? 'video'
@@ -55,25 +59,10 @@ export async function getMediaInfoActivity(params: {
? 'audio'
: isImage
? 'image'
: isDocument
: isPdf
? 'document'
: 'file'
const isPdf =
params.mediaType === 'application/pdf' ||
params.mediaType === 'text/plain' ||
params.mediaType === 'text/csv'
const proxyType: 'image' | 'video' | 'audio' | 'pdf' | undefined = isVideo
? 'video'
: isAudio
? 'audio'
: isImage
? 'image'
: isPdf || isDocument
? 'pdf'
: undefined
const mediaInfo: PrismaJson.MediaInfo = {
duration: 0,
filesize: 0,
@@ -84,7 +73,7 @@ export async function getMediaInfoActivity(params: {
videoPreview: { width: 0, height: 0 },
finishedAt: new Date().toISOString(),
metadata: null,
mimeType: params.mediaType,
mimeType: params.mediaType || '',
original: {
key: '', // Will be filled by caller or updated later
downloadUrl: '',
@@ -182,7 +171,7 @@ export async function getMediaInfoActivity(params: {
{ key: 'resolution_width', value: info.originalWidth },
{ key: 'resolution_height', value: info.originalHeight },
)
} else if (isPdf || isDocument) {
} else if (isPdf) {
const info = await transcodeService.getPdfInfo(params.filePath)
mediaInfo.frames = info.totalFrames
mediaInfo.metadata = {
@@ -18,9 +18,10 @@ export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void>
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
const { renderPdfPagesActivity } = getActivities()
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
assetKey: key,
assetKey: pdfKey,
assetId: asset.id,
start: payload.pdfPages.start,
end: payload.pdfPages.end,
@@ -43,6 +43,7 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void>
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath,
assetId: asset.id,
proxyType: 'image',
mediaType: asset.mediaType || '',
})
@@ -53,6 +53,7 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath: currentFilePath,
assetId: asset.id,
proxyType: 'pdf',
mediaType: asset.mediaType || '',
})
@@ -43,9 +43,11 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
tmpDir = download.tmpDir
const spec = task.payload?.transcode || {}
const proxyType = asset.mediaType?.startsWith('audio/') ? 'audio' : 'video'
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
filePath,
assetId: asset.id,
proxyType,
mediaType: asset.mediaType || '',
})
+2 -1
View File
@@ -51,6 +51,7 @@ interface BreadcrumbNavProps {
type: 'file' | 'folder'
version?: number
mediaType?: string | null
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
}
isRootFolder: boolean
displayStyle?: 'card' | 'list'
@@ -158,7 +159,7 @@ export function BreadcrumbNav({
}
}
const isAudio = currentAsset.mediaType?.startsWith('audio/')
const isAudio = currentAsset.proxyType === 'audio'
const hasVideoTranscodes =
!isAudio &&
downloadInfo?.videoTranscodes &&
@@ -346,7 +346,7 @@ export const MessageCard: React.FC<MessageCardProps> = ({
{message.attachments && message.attachments.length > 0 && (
<div className="flex flex-wrap gap-2 mt-3">
{message.attachments.map((att) => {
const isImage = att.mediaType?.startsWith('image/')
const isImage = att.proxyType === 'image'
const name = att.url?.split('/').pop()?.split('?')[0] || 'file'
return (
<div
@@ -4,19 +4,19 @@ import { kindOf, pickDefaultCompareVersions, clampFrame, isSameType } from './co
describe('compare-utils', () => {
describe('kindOf', () => {
it('classifies image media types', () => {
expect(kindOf('image/png')).toBe('image')
expect(kindOf('image/webp')).toBe('image')
expect(kindOf('image')).toBe('image')
expect(kindOf({ proxyType: 'image' })).toBe('image')
})
it('classifies video media types', () => {
expect(kindOf('video/mp4')).toBe('video')
expect(kindOf('video/quicktime')).toBe('video')
expect(kindOf('video')).toBe('video')
expect(kindOf({ proxyType: 'video' })).toBe('video')
})
it('returns unsupported for missing or unknown types', () => {
expect(kindOf(null)).toBe('unsupported')
expect(kindOf(undefined)).toBe('unsupported')
expect(kindOf('application/pdf')).toBe('unsupported')
expect(kindOf('pdf')).toBe('unsupported')
})
})
@@ -1,10 +1,11 @@
import type { PaneKind } from './types'
/** Classify an asset's media type into a compare pane kind. */
export function kindOf(mediaType: string | null | undefined): PaneKind {
if (!mediaType) return 'unsupported'
if (mediaType.startsWith('image/')) return 'image'
if (mediaType.startsWith('video/')) return 'video'
export function kindOf(item: { proxyType?: string | null } | string | null | undefined): PaneKind {
if (!item) return 'unsupported'
const proxyType = typeof item === 'object' ? item.proxyType : item
if (proxyType === 'image') return 'image'
if (proxyType === 'video') return 'video'
return 'unsupported'
}
@@ -214,8 +214,7 @@ export function FileListItem({
alt=""
className="w-full h-full object-cover"
/>
) : displayItem.preview?.mediaType?.startsWith('audio/') ||
displayItem.mediaType?.startsWith('audio/') ? (
) : displayItem.preview?.proxyType === 'audio' || displayItem.proxyType === 'audio' ? (
<AudioLines className="h-4 w-4 text-muted-foreground" />
) : displayItem.type === 'folder' ? (
<Folder className="h-4 w-4 text-primary" />
@@ -22,8 +22,7 @@ export const CarouselFilePreview = ({ item }: { item: AssetInfo }) => {
)
}
const isAudio =
item.preview?.mediaType?.startsWith('audio/') || item.mediaType?.startsWith('audio/')
const isAudio = item.preview?.proxyType === 'audio' || item.proxyType === 'audio'
if (isAudio) {
return (
<div className="w-full h-full flex items-center justify-center bg-muted">
@@ -315,7 +315,7 @@ export function FileViewerRightSidebar({
return (
<div className="bg-background h-full flex flex-col overflow-hidden">
{/* Lightbox */}
{viewingAttachment && viewingAttachment.mediaType?.startsWith('image/') && (
{viewingAttachment && viewingAttachment.proxyType === 'image' && (
<div
className="fixed inset-0 z-[100] bg-black/95 flex flex-col items-center justify-center p-4 animate-in fade-in duration-200"
onClick={() => setViewingAttachment(null)}
@@ -92,14 +92,13 @@ export const NotificationCard = ({ notification }: NotificationCardProps) => {
})()
const isVideoWithSprite =
asset?.mediaType?.startsWith('video/') &&
asset?.proxyType === 'video' &&
asset.preview &&
asset.thumbnailUrl &&
asset.originalWidth &&
asset.originalHeight
const hasPreview =
asset?.preview && (asset.mediaType?.startsWith('image') || asset.mediaType?.startsWith('video'))
const hasPreview = asset?.preview && (asset.proxyType === 'image' || asset.proxyType === 'video')
return (
<div className="flex py-2 cursor-pointer group border border-transparent items-center justify-center">
@@ -647,10 +647,9 @@ export function PublicShareManager({
mediaControllerRef.current?.pause()
}}
selectedCommentId={selectedCommentId}
hideAnnotationControl={(isCompareMode
? compareActiveAsset
: currentSelectedItem
)?.mediaType?.startsWith('audio/')}
hideAnnotationControl={
(isCompareMode ? compareActiveAsset : currentSelectedItem)?.proxyType === 'audio'
}
/>
</div>
</>
@@ -118,7 +118,7 @@ export const CompareVideoPane = forwardRef<ComparePaneHandle, CompareVideoPanePr
const [currentResolution, setCurrentResolution] = useState(initialRes?.resolution ?? 'Original')
const currentSrcRef = useRef(initialRes?.url)
const isAudio = file.mediaType?.startsWith('audio/')
const isAudio = file.proxyType === 'audio'
const { currentFrame, seekToFrame } = useFramePlayer(videoRef, frameRate, totalFrames, isAudio)
const zoomRef = useRef(zoom)
@@ -440,7 +440,7 @@ export const CompareVideoPane = forwardRef<ComparePaneHandle, CompareVideoPanePr
>
<div ref={videoContainerRef} className="absolute inset-0 z-[-1]" />
{file.mediaType?.startsWith('audio/') ? (
{isAudio ? (
<div className="flex flex-col items-center justify-center text-muted-foreground w-full h-full pointer-events-none select-none">
<AudioLines
className={cn(
@@ -114,7 +114,7 @@ export const VideoControlBar: React.FC<ControlBarProps> = ({
// fullscreen). Compare keeps it in-flow. Colors stay theme-aware either way.
const overlay = state.isFullScreen && floatOverlayInFullScreen
const isAudio = data.mediaType?.startsWith('audio/')
const isAudio = data.proxyType === 'audio'
const previewResolutions = resolutions.filter((r) => !r.isRaw)
const startTimecode = data.media?.metadata?.startTimecode
@@ -194,7 +194,7 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
const dbTotalFrames = metadata?.totalFrames || 0
const containerDuration = metadata?.duration || 0
const isAudio = data.mediaType?.startsWith('audio/')
const isAudio = data.proxyType === 'audio'
const totalFrames = resolveTotalFrames({ dbTotalFrames, containerDuration, frameRate })
const { currentFrame, seekToFrame } = useFramePlayer(videoRef, frameRate, totalFrames, isAudio)
@@ -626,7 +626,7 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
{/* Hidden VideoJS container */}
<div ref={videoContainerRef} className="absolute inset-0 z-[-1]" />
{data.mediaType?.startsWith('audio/') ? (
{isAudio ? (
<div className="flex flex-col items-center justify-center text-muted-foreground w-full h-full pointer-events-none select-none">
<AudioLines
className={cn(
+6
View File
@@ -32,9 +32,11 @@ export const sampleVideoAsset: AssetInfo = {
type: 'file',
status: 'ready',
mediaType: 'video',
proxyType: 'video',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
media: {
proxyType: 'video',
original: {
downloadUrl: SAMPLE_VIDEO_URL,
key: 'sample.mp4',
@@ -81,6 +83,7 @@ export const containerLongerVideoAsset: AssetInfo = {
...sampleVideoAsset,
id: 'e2e-container-longer-asset',
media: {
...sampleVideoAsset.media,
original: {
downloadUrl: SAMPLE_VIDEO_URL,
key: 'sample.mp4',
@@ -123,6 +126,7 @@ export const longAudioVideoAsset: AssetInfo = {
id: 'e2e-long-audio-asset',
name: 'sample-long-audio.mp4',
media: {
...sampleVideoAsset.media,
original: {
downloadUrl: LONG_AUDIO_VIDEO_URL,
key: 'sample-long-audio.mp4',
@@ -148,7 +152,9 @@ export const sampleAudioAsset: AssetInfo = {
id: 'e2e-sample-audio-asset',
name: 'sample-audio.mp4',
mediaType: 'audio/mp4',
proxyType: 'audio',
media: {
proxyType: 'audio',
original: {
downloadUrl: SAMPLE_AUDIO_URL,
key: 'sample-audio.mp4',
@@ -464,7 +464,7 @@ function FileViewPage() {
mediaControllerRef.current?.pause()
}}
selectedCommentId={selectedCommentId}
hideAnnotationControl={sidebarFile?.mediaType?.startsWith('audio/')}
hideAnnotationControl={sidebarFile?.proxyType === 'audio'}
/>
)}
</div>