mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 21:27:40 +00:00
feat(asset): replace mediaType/mimeType logic with proxyType
This commit is contained in:
@@ -88,8 +88,9 @@ export async function generateEmbeddingActivity(params: GenerateEmbeddingParams)
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
const isImage = asset.mediaType.startsWith('image/')
|
||||
const isVideo = asset.mediaType.startsWith('video/')
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
throw ApplicationFailure.create({
|
||||
|
||||
@@ -409,6 +409,7 @@ describe('DatabaseSessionStorage', () => {
|
||||
name: 'test.png',
|
||||
type: 'file',
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
@@ -461,6 +462,7 @@ describe('DatabaseSessionStorage', () => {
|
||||
name: 'test.mp4',
|
||||
type: 'file',
|
||||
mediaType: 'video/mp4',
|
||||
media: { proxyType: 'video' },
|
||||
projectId: project.id,
|
||||
status: 'uploaded',
|
||||
},
|
||||
|
||||
@@ -235,15 +235,14 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: assetId },
|
||||
})
|
||||
if (asset?.mediaType) {
|
||||
const type = getSimpleMediaType(asset.mediaType)
|
||||
if (type === 'image') {
|
||||
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
|
||||
} else if (type === 'video') {
|
||||
mediaTools.push(createScreenshotTool(assetId, userCommentId))
|
||||
} else if (type === 'pdf') {
|
||||
mediaTools.push(createReadPdfPagesTool(assetId, userCommentId))
|
||||
}
|
||||
const proxyType = (asset?.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const type = getSimpleMediaType(proxyType)
|
||||
if (type === 'image') {
|
||||
mediaTools.push(createAnalyzeImageTool(assetId, userCommentId))
|
||||
} else if (type === 'video') {
|
||||
mediaTools.push(createScreenshotTool(assetId, userCommentId))
|
||||
} else if (type === 'pdf') {
|
||||
mediaTools.push(createReadPdfPagesTool(assetId, userCommentId))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -77,6 +77,14 @@ export function createReadPdfPagesTool(
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (proxyType && proxyType !== 'pdf') {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Asset ${targetAssetId} is not a PDF or document.` }],
|
||||
details: {},
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch trigger comment details for comment page and annotations
|
||||
let commentTimestamp: number | null = null
|
||||
let annotations: unknown = null
|
||||
|
||||
@@ -64,6 +64,7 @@ describe('Agent Autofill Workflow', () => {
|
||||
storageKey: { key: 'asset-key' },
|
||||
project: { id: 'p1', teamId: 't1' },
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
})
|
||||
mockActivities.downloadMediaToTmpActivity.mockResolvedValue({
|
||||
filePath: '/tmp/test.png',
|
||||
@@ -290,6 +291,7 @@ describe('Agent Autofill Workflow', () => {
|
||||
storageKey: { key: 'asset-key' },
|
||||
project: { id: 'p1', teamId: 't1' },
|
||||
mediaType: 'audio/mp3',
|
||||
media: { proxyType: 'audio' },
|
||||
})
|
||||
|
||||
const task = await prisma.workflowTask.create({
|
||||
|
||||
@@ -64,8 +64,9 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise<void> {
|
||||
const projectId = asset.project.id
|
||||
|
||||
// 2. Prepare Data (Images)
|
||||
const isImage = asset.mediaType?.startsWith('image/') || false
|
||||
const isVideo = asset.mediaType?.startsWith('video/') || false
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const isImage = proxyType === 'image'
|
||||
const isVideo = proxyType === 'video'
|
||||
|
||||
if (!isImage && !isVideo) {
|
||||
throw ApplicationFailure.create({
|
||||
|
||||
@@ -25,7 +25,13 @@ describe('AgentChatPromptBuilder', () => {
|
||||
})
|
||||
|
||||
it('should build prompt with image asset details', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1').withAssetDetails('test.png', 'image/png')
|
||||
const builder = new AgentChatPromptBuilder('a1').withAssetDetails(
|
||||
'test.png',
|
||||
'image/png',
|
||||
undefined,
|
||||
undefined,
|
||||
'image',
|
||||
)
|
||||
const result = builder.build()
|
||||
|
||||
expect(result).toContain('File Name: test.png')
|
||||
@@ -39,6 +45,8 @@ describe('AgentChatPromptBuilder', () => {
|
||||
'movie.mp4',
|
||||
'video/mp4',
|
||||
12.34,
|
||||
undefined,
|
||||
'video',
|
||||
)
|
||||
const result = builder.build()
|
||||
|
||||
@@ -55,6 +63,7 @@ describe('AgentChatPromptBuilder', () => {
|
||||
'application/pdf',
|
||||
undefined,
|
||||
15,
|
||||
'pdf',
|
||||
)
|
||||
const result = builder.build()
|
||||
|
||||
@@ -69,7 +78,7 @@ describe('AgentChatPromptBuilder', () => {
|
||||
|
||||
it('should build prompt with comment page for PDF asset', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1')
|
||||
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10)
|
||||
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
|
||||
.withCommentTimestamp(3)
|
||||
const result = builder.build()
|
||||
|
||||
@@ -79,7 +88,7 @@ describe('AgentChatPromptBuilder', () => {
|
||||
it('should build prompt for PDF continuation mode with comment page', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1')
|
||||
.withContinuation(true)
|
||||
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10)
|
||||
.withAssetDetails('doc.pdf', 'application/pdf', undefined, 10, 'pdf')
|
||||
.withCommentTimestamp(4)
|
||||
const result = builder.build()
|
||||
|
||||
@@ -98,7 +107,7 @@ describe('AgentChatPromptBuilder', () => {
|
||||
const builder = new AgentChatPromptBuilder('a1')
|
||||
.withExplicitMention(true)
|
||||
.withPathContext('src/main.ts')
|
||||
.withAssetDetails('movie.mp4', 'video/mp4', 12.34)
|
||||
.withAssetDetails('movie.mp4', 'video/mp4', 12.34, undefined, 'video')
|
||||
.withCommentTimestamp(8.2)
|
||||
const result = builder.build()
|
||||
|
||||
|
||||
@@ -1,19 +1,7 @@
|
||||
export function getSimpleMediaType(mediaType?: string): 'image' | 'video' | 'pdf' | 'other' {
|
||||
if (!mediaType) return 'other'
|
||||
const lower = mediaType.toLowerCase()
|
||||
if (
|
||||
lower.startsWith('image/') ||
|
||||
lower === 'application/x-photoshop' ||
|
||||
lower === 'image/vnd.adobe.photoshop'
|
||||
) {
|
||||
return 'image'
|
||||
}
|
||||
if (lower.startsWith('video/')) {
|
||||
return 'video'
|
||||
}
|
||||
if (lower === 'application/pdf' || lower.endsWith('.pdf')) {
|
||||
return 'pdf'
|
||||
}
|
||||
export function getSimpleMediaType(proxyType?: string | null): 'image' | 'video' | 'pdf' | 'other' {
|
||||
if (proxyType === 'image') return 'image'
|
||||
if (proxyType === 'video') return 'video'
|
||||
if (proxyType === 'pdf') return 'pdf'
|
||||
return 'other'
|
||||
}
|
||||
|
||||
@@ -21,6 +9,7 @@ export class AgentChatPromptBuilder {
|
||||
private assetId: string
|
||||
private assetName?: string
|
||||
private mediaType?: string
|
||||
private proxyType?: string
|
||||
private videoDuration?: number
|
||||
private totalPages?: number
|
||||
private commentTimestamp?: number
|
||||
@@ -64,9 +53,11 @@ export class AgentChatPromptBuilder {
|
||||
mediaType: string | null,
|
||||
duration?: number,
|
||||
totalPages?: number,
|
||||
proxyType?: string | null,
|
||||
): this {
|
||||
this.assetName = name
|
||||
this.mediaType = mediaType || undefined
|
||||
this.proxyType = proxyType || undefined
|
||||
this.videoDuration = duration
|
||||
this.totalPages = totalPages
|
||||
return this
|
||||
@@ -85,10 +76,12 @@ export class AgentChatPromptBuilder {
|
||||
}
|
||||
|
||||
build(): string {
|
||||
const effectiveType = this.proxyType || this.mediaType
|
||||
|
||||
if (this.isContinuation) {
|
||||
let instruction = ''
|
||||
if (this.commentTimestamp !== undefined) {
|
||||
const type = getSimpleMediaType(this.mediaType)
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
if (type === 'pdf') {
|
||||
instruction += `Comment Page: ${Math.round(this.commentTimestamp)}`
|
||||
} else {
|
||||
@@ -111,8 +104,8 @@ export class AgentChatPromptBuilder {
|
||||
if (this.assetName) {
|
||||
instruction += `\nFile Name: ${this.assetName}`
|
||||
}
|
||||
if (this.mediaType) {
|
||||
const type = getSimpleMediaType(this.mediaType)
|
||||
if (effectiveType) {
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
instruction += `\nFile Type: ${type}`
|
||||
if (type === 'video' && this.videoDuration !== undefined) {
|
||||
instruction += `\nVideo Length: ${this.videoDuration.toFixed(2)} seconds`
|
||||
@@ -121,7 +114,7 @@ export class AgentChatPromptBuilder {
|
||||
}
|
||||
}
|
||||
if (this.commentTimestamp !== undefined) {
|
||||
const type = getSimpleMediaType(this.mediaType)
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
if (type === 'pdf') {
|
||||
instruction += `\nComment Page: ${Math.round(this.commentTimestamp)}`
|
||||
} else {
|
||||
@@ -142,8 +135,8 @@ export class AgentChatPromptBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
if (this.mediaType) {
|
||||
const type = getSimpleMediaType(this.mediaType)
|
||||
if (effectiveType) {
|
||||
const type = getSimpleMediaType(effectiveType)
|
||||
if (type === 'image') {
|
||||
instruction += `\n\nIf you need to view the image data, call the 'analyze_image' tool. It does not require any parameters.`
|
||||
} else if (type === 'video') {
|
||||
|
||||
@@ -61,6 +61,7 @@ describe('Agent Chat Workflow', () => {
|
||||
type: 'file',
|
||||
name: 'test-file.png',
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
parentId: 'parent-folder-id',
|
||||
})
|
||||
mockActivities.getCommentActivity.mockResolvedValue({
|
||||
@@ -133,7 +134,7 @@ describe('Agent Chat Workflow', () => {
|
||||
// Verify agent instruction composition
|
||||
const expectedInstruction1 = new AgentChatPromptBuilder('a1')
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined)
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
|
||||
.withCommentTimestamp(null)
|
||||
.withExplicitMention(true)
|
||||
.build()
|
||||
@@ -239,7 +240,7 @@ describe('Agent Chat Workflow', () => {
|
||||
// Verify instruction contains not explicitly mentioned instructions
|
||||
const expectedInstruction3 = new AgentChatPromptBuilder('a1')
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined)
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
|
||||
.withCommentTimestamp(null)
|
||||
.withExplicitMention(false)
|
||||
.build()
|
||||
@@ -271,18 +272,21 @@ describe('Agent Chat Workflow', () => {
|
||||
{
|
||||
asset: {
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
storageKey: { key: 'attachments/image1.png' },
|
||||
},
|
||||
},
|
||||
{
|
||||
asset: {
|
||||
mediaType: 'video/mp4', // non-image, should be ignored
|
||||
media: { proxyType: 'video' },
|
||||
storageKey: { key: 'attachments/video.mp4' },
|
||||
},
|
||||
},
|
||||
{
|
||||
asset: {
|
||||
mediaType: 'image/jpeg',
|
||||
media: { proxyType: 'image' },
|
||||
storageKey: { key: 'attachments/image2.jpg' },
|
||||
},
|
||||
},
|
||||
@@ -305,7 +309,7 @@ describe('Agent Chat Workflow', () => {
|
||||
|
||||
const expectedInstruction4 = new AgentChatPromptBuilder('a1')
|
||||
.withPathContext('Path: folder/subfolder/file.png')
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined)
|
||||
.withAssetDetails('test-file.png', 'image/png', undefined, undefined, 'image')
|
||||
.withCommentTimestamp(null)
|
||||
.withExplicitMention(false)
|
||||
.build()
|
||||
|
||||
@@ -74,7 +74,8 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
// 5. Prepare Images (Attachments only)
|
||||
if (userComment.attachments) {
|
||||
for (const att of userComment.attachments) {
|
||||
if (att.asset?.mediaType?.startsWith('image/') && att.asset?.storageKey?.key) {
|
||||
const attProxyType = (att.asset?.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
if (attProxyType === 'image' && att.asset?.storageKey?.key) {
|
||||
attachmentImageUrls.push(att.asset.storageKey.key)
|
||||
}
|
||||
}
|
||||
@@ -161,10 +162,11 @@ export async function agentChat(task: WorkflowTask): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
const proxyType = (asset.media as PrismaJson.MediaInfo | null)?.proxyType
|
||||
const instruction = new AgentChatPromptBuilder(asset.id)
|
||||
.withContinuation(!isNewChat)
|
||||
.withPathContext(pathContext)
|
||||
.withAssetDetails(asset.name, asset.mediaType, duration, totalPages)
|
||||
.withAssetDetails(asset.name, asset.mediaType, duration, totalPages, proxyType)
|
||||
.withCommentTimestamp(commentTimestamp)
|
||||
.withExplicitMention(payload.agent?.explicitMention)
|
||||
.withAttachedFiles(attachedFileDetailsList)
|
||||
|
||||
@@ -2000,8 +2000,10 @@ export class AssetService {
|
||||
private async toPreviewInfo(asset: Asset | AssetWithIncludes): Promise<PreviewInfo | null> {
|
||||
if (!asset.media) return null
|
||||
|
||||
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
|
||||
let thumbnailUrl = undefined
|
||||
if (asset.mediaType?.startsWith('image/') && asset.media.thumbnail?.key) {
|
||||
if (proxyType === 'image' && asset.media.thumbnail?.key) {
|
||||
thumbnailUrl = await s3Service.presign(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
asset.media.thumbnail.key,
|
||||
@@ -2024,8 +2026,6 @@ export class AssetService {
|
||||
)
|
||||
}
|
||||
|
||||
const proxyType = (asset.media?.proxyType || null) as 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
|
||||
return {
|
||||
mediaType: asset.mediaType,
|
||||
proxyType,
|
||||
@@ -2057,11 +2057,18 @@ export class AssetService {
|
||||
a.asset.storageKey.key,
|
||||
'GET',
|
||||
)
|
||||
const attachmentProxyType = (a.asset.media?.proxyType || null) as
|
||||
| 'image'
|
||||
| 'video'
|
||||
| 'audio'
|
||||
| 'pdf'
|
||||
| null
|
||||
attachments.push({
|
||||
id: a.id,
|
||||
assetId: a.asset.id,
|
||||
url,
|
||||
mediaType: a.asset.mediaType,
|
||||
proxyType: attachmentProxyType,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,6 +181,7 @@ describe('ChatService', () => {
|
||||
name: 'doc.jpg',
|
||||
type: 'file',
|
||||
mediaType: 'image/jpeg',
|
||||
media: { proxyType: 'image' },
|
||||
status: 'processed',
|
||||
projectId: project.id,
|
||||
parentId: childFolder.id,
|
||||
|
||||
@@ -232,7 +232,8 @@ export class ChatService {
|
||||
where: { id: fileId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
if (file && file.mediaType?.startsWith('image/') && file.storageKey?.key) {
|
||||
const isImage = (file?.media as PrismaJson.MediaInfo | null)?.proxyType === 'image'
|
||||
if (file && isImage && file.storageKey?.key) {
|
||||
resolvedImageUrls.push(file.storageKey.key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,6 +402,7 @@ export class NotificationService {
|
||||
id: a.id,
|
||||
name: a.name,
|
||||
mediaType: a.mediaType || undefined,
|
||||
proxyType: (a.media as PrismaJson.MediaInfo | null)?.proxyType || undefined,
|
||||
preview: previewUrl || undefined,
|
||||
thumbnailUrl: thumbnailUrl || undefined,
|
||||
originalWidth: originalWidth || undefined,
|
||||
|
||||
@@ -22,6 +22,7 @@ import { ImageTranscoder, PdfTranscoder, VideoTranscoder } from '@shumai/transco
|
||||
import { generateKeyBetween } from 'jittered-fractional-indexing'
|
||||
import { ulid } from 'ulid'
|
||||
import { sanitizeFilename } from '@shumai/core/src/utils/filename'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
|
||||
export class UploadService {
|
||||
constructor(private readonly prismaClient: typeof prisma = prisma) {}
|
||||
@@ -300,16 +301,14 @@ export class UploadService {
|
||||
},
|
||||
})
|
||||
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
const isImage = asset.mediaType?.startsWith('image/')
|
||||
const isAudio = asset.mediaType?.startsWith('audio/')
|
||||
const isPdf =
|
||||
asset.mediaType === 'application/pdf' ||
|
||||
asset.mediaType === 'text/plain' ||
|
||||
asset.mediaType === 'text/csv' ||
|
||||
asset.name.toLowerCase().endsWith('.pdf') ||
|
||||
asset.name.toLowerCase().endsWith('.txt') ||
|
||||
asset.name.toLowerCase().endsWith('.csv')
|
||||
const proxyType =
|
||||
(asset.media as PrismaJson.MediaInfo | null)?.proxyType ||
|
||||
getProxyType(asset.mediaType, asset.name)
|
||||
|
||||
const isVideo = proxyType === 'video'
|
||||
const isImage = proxyType === 'image'
|
||||
const isAudio = proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
|
||||
if (autofillAgent && (isVideo || isImage)) {
|
||||
await tx.workflowTask.create({
|
||||
@@ -327,7 +326,7 @@ export class UploadService {
|
||||
})
|
||||
}
|
||||
|
||||
if (!isVideo && !isImage && !isAudio && !isPdf) {
|
||||
if (!proxyType) {
|
||||
await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { detectSupportedMimeType } from './mime'
|
||||
import { detectSupportedMimeType, getProxyType } from './mime'
|
||||
|
||||
describe('detectSupportedMimeType', () => {
|
||||
it('should detect JPEG', () => {
|
||||
@@ -39,3 +39,31 @@ describe('detectSupportedMimeType', () => {
|
||||
expect(detectSupportedMimeType(buffer)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getProxyType', () => {
|
||||
it('should detect image proxyType', () => {
|
||||
expect(getProxyType('image/png', 'test.png')).toBe('image')
|
||||
expect(getProxyType('image/jpeg', 'photo.jpg')).toBe('image')
|
||||
})
|
||||
|
||||
it('should detect video proxyType', () => {
|
||||
expect(getProxyType('video/mp4', 'clip.mp4')).toBe('video')
|
||||
})
|
||||
|
||||
it('should detect audio proxyType', () => {
|
||||
expect(getProxyType('audio/mpeg', 'song.mp3')).toBe('audio')
|
||||
})
|
||||
|
||||
it('should detect pdf proxyType for pdf, csv, txt, and doc files', () => {
|
||||
expect(getProxyType('application/pdf', 'doc.pdf')).toBe('pdf')
|
||||
expect(getProxyType('text/plain', 'notes.txt')).toBe('pdf')
|
||||
expect(getProxyType('text/csv', 'data.csv')).toBe('pdf')
|
||||
expect(getProxyType('application/msword', 'letter.doc')).toBe('pdf')
|
||||
expect(getProxyType(null, 'file.txt')).toBe('pdf')
|
||||
})
|
||||
|
||||
it('should return null for unsupported files', () => {
|
||||
expect(getProxyType('application/zip', 'archive.zip')).toBeNull()
|
||||
expect(getProxyType(null, 'unknown.bin')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,3 +1,40 @@
|
||||
export type ProxyType = 'image' | 'video' | 'audio' | 'pdf'
|
||||
|
||||
export function getProxyType(
|
||||
mediaType?: string | null,
|
||||
filename?: string | null,
|
||||
): ProxyType | null {
|
||||
const lowerMediaType = mediaType?.toLowerCase() || ''
|
||||
const lowerFilename = filename?.toLowerCase() || ''
|
||||
|
||||
if (lowerMediaType.startsWith('image/')) return 'image'
|
||||
if (lowerMediaType.startsWith('video/')) return 'video'
|
||||
if (lowerMediaType.startsWith('audio/')) return 'audio'
|
||||
|
||||
if (
|
||||
lowerMediaType === 'application/pdf' ||
|
||||
lowerMediaType === 'text/plain' ||
|
||||
lowerMediaType === 'text/csv' ||
|
||||
lowerMediaType.includes('msword') ||
|
||||
lowerMediaType.includes('officedocument') ||
|
||||
lowerMediaType.includes('vnd.ms-') ||
|
||||
lowerMediaType.startsWith('text/') ||
|
||||
lowerFilename.endsWith('.pdf') ||
|
||||
lowerFilename.endsWith('.txt') ||
|
||||
lowerFilename.endsWith('.csv') ||
|
||||
lowerFilename.endsWith('.doc') ||
|
||||
lowerFilename.endsWith('.docx') ||
|
||||
lowerFilename.endsWith('.xls') ||
|
||||
lowerFilename.endsWith('.xlsx') ||
|
||||
lowerFilename.endsWith('.ppt') ||
|
||||
lowerFilename.endsWith('.pptx')
|
||||
) {
|
||||
return 'pdf'
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
|
||||
|
||||
export function detectSupportedMimeType(buffer: Uint8Array): string | null {
|
||||
|
||||
@@ -191,6 +191,7 @@ export const attachmentInfoSchema = z.object({
|
||||
assetId: z.string(),
|
||||
url: z.string(),
|
||||
mediaType: z.string().nullable(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
})
|
||||
export type AttachmentInfo = z.infer<typeof attachmentInfoSchema>
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ export type EntityInfo = z.infer<typeof entityInfoSchema>
|
||||
export const assetPreviewSchema = z.object({
|
||||
id: z.string().optional(),
|
||||
mediaType: z.string().optional(),
|
||||
proxyType: z.enum(['image', 'video', 'audio', 'pdf']).nullable().optional(),
|
||||
name: z.string().optional(),
|
||||
preview: z.string().optional(),
|
||||
thumbnailUrl: z.string().optional(),
|
||||
|
||||
@@ -230,6 +230,7 @@ describe.each(['local', 'temporal'] as const)(
|
||||
type: 'file',
|
||||
status: 'uploaded',
|
||||
mediaType: 'image/png',
|
||||
media: { proxyType: 'image' },
|
||||
projectId: project.id,
|
||||
storageKeyId: storageKey.id,
|
||||
},
|
||||
|
||||
@@ -117,6 +117,7 @@ describe('Transcode Activities', () => {
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/v.mp4',
|
||||
proxyType: 'video',
|
||||
mediaType: 'video/mp4',
|
||||
})
|
||||
|
||||
@@ -153,6 +154,7 @@ describe('Transcode Activities', () => {
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/i.png',
|
||||
proxyType: 'image',
|
||||
mediaType: 'image/png',
|
||||
})
|
||||
|
||||
@@ -188,6 +190,7 @@ describe('Transcode Activities', () => {
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/a.mp3',
|
||||
proxyType: 'audio',
|
||||
mediaType: 'audio/mpeg',
|
||||
})
|
||||
|
||||
@@ -221,6 +224,7 @@ describe('Transcode Activities', () => {
|
||||
await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/d.pdf',
|
||||
proxyType: 'pdf',
|
||||
mediaType: 'application/pdf',
|
||||
})
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import { s3Service } from '@shumai/core/src/s3/s3'
|
||||
import { transcodeService } from '@shumai/core/src/transcode/transcode'
|
||||
import { metadataService } from '@shumai/core/src/metadata/metadata'
|
||||
import { stemFromKey } from '@shumai/core/src/utils/filename'
|
||||
import { getProxyType } from '@shumai/core/src/utils/mime'
|
||||
import { ApplicationFailure } from '@temporalio/activity'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
@@ -24,7 +25,12 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
|
||||
const record = err as Record<string, unknown>
|
||||
return {
|
||||
code: typeof record.code === 'string' ? record.code : undefined,
|
||||
message: typeof record.message === 'string' ? record.message : String(err),
|
||||
message:
|
||||
typeof record.message === 'string'
|
||||
? record.message
|
||||
: err instanceof Error
|
||||
? err.message
|
||||
: String(err),
|
||||
name: typeof record.name === 'string' ? record.name : undefined,
|
||||
}
|
||||
}
|
||||
@@ -36,18 +42,16 @@ function getErrorDetails(err: unknown): { code?: string; message: string; name?:
|
||||
export async function getMediaInfoActivity(params: {
|
||||
filePath: string
|
||||
assetId: string
|
||||
mediaType: string
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
mediaType?: string
|
||||
}): Promise<PrismaJson.MediaInfo> {
|
||||
try {
|
||||
const isVideo = params.mediaType.startsWith('video/')
|
||||
const isImage = params.mediaType.startsWith('image/')
|
||||
const isAudio = params.mediaType.startsWith('audio/')
|
||||
const isDocument =
|
||||
params.mediaType.startsWith('text/') ||
|
||||
params.mediaType === 'application/pdf' ||
|
||||
params.mediaType.includes('msword') ||
|
||||
params.mediaType.includes('officedocument') ||
|
||||
params.mediaType.includes('vnd.ms-')
|
||||
const proxyType =
|
||||
params.proxyType || getProxyType(params.mediaType, params.filePath) || undefined
|
||||
const isVideo = proxyType === 'video'
|
||||
const isImage = proxyType === 'image'
|
||||
const isAudio = proxyType === 'audio'
|
||||
const isPdf = proxyType === 'pdf'
|
||||
|
||||
const fileType = isVideo
|
||||
? 'video'
|
||||
@@ -55,25 +59,10 @@ export async function getMediaInfoActivity(params: {
|
||||
? 'audio'
|
||||
: isImage
|
||||
? 'image'
|
||||
: isDocument
|
||||
: isPdf
|
||||
? 'document'
|
||||
: 'file'
|
||||
|
||||
const isPdf =
|
||||
params.mediaType === 'application/pdf' ||
|
||||
params.mediaType === 'text/plain' ||
|
||||
params.mediaType === 'text/csv'
|
||||
|
||||
const proxyType: 'image' | 'video' | 'audio' | 'pdf' | undefined = isVideo
|
||||
? 'video'
|
||||
: isAudio
|
||||
? 'audio'
|
||||
: isImage
|
||||
? 'image'
|
||||
: isPdf || isDocument
|
||||
? 'pdf'
|
||||
: undefined
|
||||
|
||||
const mediaInfo: PrismaJson.MediaInfo = {
|
||||
duration: 0,
|
||||
filesize: 0,
|
||||
@@ -84,7 +73,7 @@ export async function getMediaInfoActivity(params: {
|
||||
videoPreview: { width: 0, height: 0 },
|
||||
finishedAt: new Date().toISOString(),
|
||||
metadata: null,
|
||||
mimeType: params.mediaType,
|
||||
mimeType: params.mediaType || '',
|
||||
original: {
|
||||
key: '', // Will be filled by caller or updated later
|
||||
downloadUrl: '',
|
||||
@@ -182,7 +171,7 @@ export async function getMediaInfoActivity(params: {
|
||||
{ key: 'resolution_width', value: info.originalWidth },
|
||||
{ key: 'resolution_height', value: info.originalHeight },
|
||||
)
|
||||
} else if (isPdf || isDocument) {
|
||||
} else if (isPdf) {
|
||||
const info = await transcodeService.getPdfInfo(params.filePath)
|
||||
mediaInfo.frames = info.totalFrames
|
||||
mediaInfo.metadata = {
|
||||
|
||||
@@ -18,9 +18,10 @@ export async function renderPdfPagesWorkflow(task: WorkflowTask): Promise<void>
|
||||
|
||||
const { asset, key } = await fetchAssetWithKey(workerQueue, task.assetId)
|
||||
const { renderPdfPagesActivity } = getActivities()
|
||||
const pdfKey = (asset.media as PrismaJson.MediaInfo | null)?.pdfTranscode?.key || key
|
||||
|
||||
const pages = await executeActivity(workerQueue, renderPdfPagesActivity, {
|
||||
assetKey: key,
|
||||
assetKey: pdfKey,
|
||||
assetId: asset.id,
|
||||
start: payload.pdfPages.start,
|
||||
end: payload.pdfPages.end,
|
||||
|
||||
@@ -43,6 +43,7 @@ export async function transcodeImageWorkflow(task: WorkflowTask): Promise<void>
|
||||
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
|
||||
filePath,
|
||||
assetId: asset.id,
|
||||
proxyType: 'image',
|
||||
mediaType: asset.mediaType || '',
|
||||
})
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ export async function transcodePdfWorkflow(task: WorkflowTask): Promise<void> {
|
||||
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
|
||||
filePath: currentFilePath,
|
||||
assetId: asset.id,
|
||||
proxyType: 'pdf',
|
||||
mediaType: asset.mediaType || '',
|
||||
})
|
||||
|
||||
|
||||
@@ -43,9 +43,11 @@ export async function transcodeVideoWorkflow(task: WorkflowTask): Promise<void>
|
||||
tmpDir = download.tmpDir
|
||||
|
||||
const spec = task.payload?.transcode || {}
|
||||
const proxyType = asset.mediaType?.startsWith('audio/') ? 'audio' : 'video'
|
||||
const mediaInfo = await executeActivity(workerQueue, getMediaInfoActivity, {
|
||||
filePath,
|
||||
assetId: asset.id,
|
||||
proxyType,
|
||||
mediaType: asset.mediaType || '',
|
||||
})
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ interface BreadcrumbNavProps {
|
||||
type: 'file' | 'folder'
|
||||
version?: number
|
||||
mediaType?: string | null
|
||||
proxyType?: 'image' | 'video' | 'audio' | 'pdf' | null
|
||||
}
|
||||
isRootFolder: boolean
|
||||
displayStyle?: 'card' | 'list'
|
||||
@@ -158,7 +159,7 @@ export function BreadcrumbNav({
|
||||
}
|
||||
}
|
||||
|
||||
const isAudio = currentAsset.mediaType?.startsWith('audio/')
|
||||
const isAudio = currentAsset.proxyType === 'audio'
|
||||
const hasVideoTranscodes =
|
||||
!isAudio &&
|
||||
downloadInfo?.videoTranscodes &&
|
||||
|
||||
@@ -346,7 +346,7 @@ export const MessageCard: React.FC<MessageCardProps> = ({
|
||||
{message.attachments && message.attachments.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 mt-3">
|
||||
{message.attachments.map((att) => {
|
||||
const isImage = att.mediaType?.startsWith('image/')
|
||||
const isImage = att.proxyType === 'image'
|
||||
const name = att.url?.split('/').pop()?.split('?')[0] || 'file'
|
||||
return (
|
||||
<div
|
||||
|
||||
@@ -4,19 +4,19 @@ import { kindOf, pickDefaultCompareVersions, clampFrame, isSameType } from './co
|
||||
describe('compare-utils', () => {
|
||||
describe('kindOf', () => {
|
||||
it('classifies image media types', () => {
|
||||
expect(kindOf('image/png')).toBe('image')
|
||||
expect(kindOf('image/webp')).toBe('image')
|
||||
expect(kindOf('image')).toBe('image')
|
||||
expect(kindOf({ proxyType: 'image' })).toBe('image')
|
||||
})
|
||||
|
||||
it('classifies video media types', () => {
|
||||
expect(kindOf('video/mp4')).toBe('video')
|
||||
expect(kindOf('video/quicktime')).toBe('video')
|
||||
expect(kindOf('video')).toBe('video')
|
||||
expect(kindOf({ proxyType: 'video' })).toBe('video')
|
||||
})
|
||||
|
||||
it('returns unsupported for missing or unknown types', () => {
|
||||
expect(kindOf(null)).toBe('unsupported')
|
||||
expect(kindOf(undefined)).toBe('unsupported')
|
||||
expect(kindOf('application/pdf')).toBe('unsupported')
|
||||
expect(kindOf('pdf')).toBe('unsupported')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import type { PaneKind } from './types'
|
||||
|
||||
/** Classify an asset's media type into a compare pane kind. */
|
||||
export function kindOf(mediaType: string | null | undefined): PaneKind {
|
||||
if (!mediaType) return 'unsupported'
|
||||
if (mediaType.startsWith('image/')) return 'image'
|
||||
if (mediaType.startsWith('video/')) return 'video'
|
||||
export function kindOf(item: { proxyType?: string | null } | string | null | undefined): PaneKind {
|
||||
if (!item) return 'unsupported'
|
||||
const proxyType = typeof item === 'object' ? item.proxyType : item
|
||||
if (proxyType === 'image') return 'image'
|
||||
if (proxyType === 'video') return 'video'
|
||||
return 'unsupported'
|
||||
}
|
||||
|
||||
|
||||
@@ -214,8 +214,7 @@ export function FileListItem({
|
||||
alt=""
|
||||
className="w-full h-full object-cover"
|
||||
/>
|
||||
) : displayItem.preview?.mediaType?.startsWith('audio/') ||
|
||||
displayItem.mediaType?.startsWith('audio/') ? (
|
||||
) : displayItem.preview?.proxyType === 'audio' || displayItem.proxyType === 'audio' ? (
|
||||
<AudioLines className="h-4 w-4 text-muted-foreground" />
|
||||
) : displayItem.type === 'folder' ? (
|
||||
<Folder className="h-4 w-4 text-primary" />
|
||||
|
||||
@@ -22,8 +22,7 @@ export const CarouselFilePreview = ({ item }: { item: AssetInfo }) => {
|
||||
)
|
||||
}
|
||||
|
||||
const isAudio =
|
||||
item.preview?.mediaType?.startsWith('audio/') || item.mediaType?.startsWith('audio/')
|
||||
const isAudio = item.preview?.proxyType === 'audio' || item.proxyType === 'audio'
|
||||
if (isAudio) {
|
||||
return (
|
||||
<div className="w-full h-full flex items-center justify-center bg-muted">
|
||||
|
||||
@@ -315,7 +315,7 @@ export function FileViewerRightSidebar({
|
||||
return (
|
||||
<div className="bg-background h-full flex flex-col overflow-hidden">
|
||||
{/* Lightbox */}
|
||||
{viewingAttachment && viewingAttachment.mediaType?.startsWith('image/') && (
|
||||
{viewingAttachment && viewingAttachment.proxyType === 'image' && (
|
||||
<div
|
||||
className="fixed inset-0 z-[100] bg-black/95 flex flex-col items-center justify-center p-4 animate-in fade-in duration-200"
|
||||
onClick={() => setViewingAttachment(null)}
|
||||
|
||||
@@ -92,14 +92,13 @@ export const NotificationCard = ({ notification }: NotificationCardProps) => {
|
||||
})()
|
||||
|
||||
const isVideoWithSprite =
|
||||
asset?.mediaType?.startsWith('video/') &&
|
||||
asset?.proxyType === 'video' &&
|
||||
asset.preview &&
|
||||
asset.thumbnailUrl &&
|
||||
asset.originalWidth &&
|
||||
asset.originalHeight
|
||||
|
||||
const hasPreview =
|
||||
asset?.preview && (asset.mediaType?.startsWith('image') || asset.mediaType?.startsWith('video'))
|
||||
const hasPreview = asset?.preview && (asset.proxyType === 'image' || asset.proxyType === 'video')
|
||||
|
||||
return (
|
||||
<div className="flex py-2 cursor-pointer group border border-transparent items-center justify-center">
|
||||
|
||||
@@ -647,10 +647,9 @@ export function PublicShareManager({
|
||||
mediaControllerRef.current?.pause()
|
||||
}}
|
||||
selectedCommentId={selectedCommentId}
|
||||
hideAnnotationControl={(isCompareMode
|
||||
? compareActiveAsset
|
||||
: currentSelectedItem
|
||||
)?.mediaType?.startsWith('audio/')}
|
||||
hideAnnotationControl={
|
||||
(isCompareMode ? compareActiveAsset : currentSelectedItem)?.proxyType === 'audio'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
|
||||
@@ -118,7 +118,7 @@ export const CompareVideoPane = forwardRef<ComparePaneHandle, CompareVideoPanePr
|
||||
const [currentResolution, setCurrentResolution] = useState(initialRes?.resolution ?? 'Original')
|
||||
const currentSrcRef = useRef(initialRes?.url)
|
||||
|
||||
const isAudio = file.mediaType?.startsWith('audio/')
|
||||
const isAudio = file.proxyType === 'audio'
|
||||
const { currentFrame, seekToFrame } = useFramePlayer(videoRef, frameRate, totalFrames, isAudio)
|
||||
|
||||
const zoomRef = useRef(zoom)
|
||||
@@ -440,7 +440,7 @@ export const CompareVideoPane = forwardRef<ComparePaneHandle, CompareVideoPanePr
|
||||
>
|
||||
<div ref={videoContainerRef} className="absolute inset-0 z-[-1]" />
|
||||
|
||||
{file.mediaType?.startsWith('audio/') ? (
|
||||
{isAudio ? (
|
||||
<div className="flex flex-col items-center justify-center text-muted-foreground w-full h-full pointer-events-none select-none">
|
||||
<AudioLines
|
||||
className={cn(
|
||||
|
||||
@@ -114,7 +114,7 @@ export const VideoControlBar: React.FC<ControlBarProps> = ({
|
||||
// fullscreen). Compare keeps it in-flow. Colors stay theme-aware either way.
|
||||
const overlay = state.isFullScreen && floatOverlayInFullScreen
|
||||
|
||||
const isAudio = data.mediaType?.startsWith('audio/')
|
||||
const isAudio = data.proxyType === 'audio'
|
||||
const previewResolutions = resolutions.filter((r) => !r.isRaw)
|
||||
|
||||
const startTimecode = data.media?.metadata?.startTimecode
|
||||
|
||||
@@ -194,7 +194,7 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
|
||||
const dbTotalFrames = metadata?.totalFrames || 0
|
||||
const containerDuration = metadata?.duration || 0
|
||||
|
||||
const isAudio = data.mediaType?.startsWith('audio/')
|
||||
const isAudio = data.proxyType === 'audio'
|
||||
const totalFrames = resolveTotalFrames({ dbTotalFrames, containerDuration, frameRate })
|
||||
const { currentFrame, seekToFrame } = useFramePlayer(videoRef, frameRate, totalFrames, isAudio)
|
||||
|
||||
@@ -626,7 +626,7 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
|
||||
{/* Hidden VideoJS container */}
|
||||
<div ref={videoContainerRef} className="absolute inset-0 z-[-1]" />
|
||||
|
||||
{data.mediaType?.startsWith('audio/') ? (
|
||||
{isAudio ? (
|
||||
<div className="flex flex-col items-center justify-center text-muted-foreground w-full h-full pointer-events-none select-none">
|
||||
<AudioLines
|
||||
className={cn(
|
||||
|
||||
@@ -32,9 +32,11 @@ export const sampleVideoAsset: AssetInfo = {
|
||||
type: 'file',
|
||||
status: 'ready',
|
||||
mediaType: 'video',
|
||||
proxyType: 'video',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
media: {
|
||||
proxyType: 'video',
|
||||
original: {
|
||||
downloadUrl: SAMPLE_VIDEO_URL,
|
||||
key: 'sample.mp4',
|
||||
@@ -81,6 +83,7 @@ export const containerLongerVideoAsset: AssetInfo = {
|
||||
...sampleVideoAsset,
|
||||
id: 'e2e-container-longer-asset',
|
||||
media: {
|
||||
...sampleVideoAsset.media,
|
||||
original: {
|
||||
downloadUrl: SAMPLE_VIDEO_URL,
|
||||
key: 'sample.mp4',
|
||||
@@ -123,6 +126,7 @@ export const longAudioVideoAsset: AssetInfo = {
|
||||
id: 'e2e-long-audio-asset',
|
||||
name: 'sample-long-audio.mp4',
|
||||
media: {
|
||||
...sampleVideoAsset.media,
|
||||
original: {
|
||||
downloadUrl: LONG_AUDIO_VIDEO_URL,
|
||||
key: 'sample-long-audio.mp4',
|
||||
@@ -148,7 +152,9 @@ export const sampleAudioAsset: AssetInfo = {
|
||||
id: 'e2e-sample-audio-asset',
|
||||
name: 'sample-audio.mp4',
|
||||
mediaType: 'audio/mp4',
|
||||
proxyType: 'audio',
|
||||
media: {
|
||||
proxyType: 'audio',
|
||||
original: {
|
||||
downloadUrl: SAMPLE_AUDIO_URL,
|
||||
key: 'sample-audio.mp4',
|
||||
|
||||
@@ -464,7 +464,7 @@ function FileViewPage() {
|
||||
mediaControllerRef.current?.pause()
|
||||
}}
|
||||
selectedCommentId={selectedCommentId}
|
||||
hideAnnotationControl={sidebarFile?.mediaType?.startsWith('audio/')}
|
||||
hideAnnotationControl={sidebarFile?.proxyType === 'audio'}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user