mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 20:37:40 +00:00
feat(transcode): add audio file support with AAC stereo mp4 proxy and styled viewer
This commit is contained in:
@@ -270,4 +270,81 @@ describe('TranscodeService', () => {
|
||||
expect(task.payload?.projectId).toBe('proj-123')
|
||||
expect(task.payload?.transcode?.videoStrategy).toBe('best_match')
|
||||
})
|
||||
|
||||
it('should parse audio info correctly from ffprobe output', async () => {
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
const mockOutput = JSON.stringify({
|
||||
format: { duration: '120.4', bit_rate: '256000' },
|
||||
streams: [
|
||||
{
|
||||
codec_type: 'audio',
|
||||
codec_name: 'flac',
|
||||
channels: 6,
|
||||
sample_rate: '44100',
|
||||
bits_per_sample: '24',
|
||||
tags: {
|
||||
mime_codec_string: 'fLaC',
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
/* eslint-enable @typescript-eslint/naming-convention */
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation((file: string, args: string[], cb: any) => {
|
||||
cb(null, { stdout: mockOutput, stderr: '' })
|
||||
})
|
||||
|
||||
const info = await transcodeService.getAudioInfo('test.flac')
|
||||
|
||||
expect(info.originalWidth).toBe(0)
|
||||
expect(info.originalHeight).toBe(0)
|
||||
expect(info.duration).toBe(120.4)
|
||||
expect(info.frameRate).toBe(0)
|
||||
expect(info.hasAudio).toBe(true)
|
||||
expect(info.videoCodec).toBeUndefined()
|
||||
expect(info.audioCodec).toBe('Fres Lossless Audio Codec')
|
||||
expect(info.audioChannels).toBe(6)
|
||||
expect(info.audioSampleRate).toBe(44100)
|
||||
expect(info.audioBitDepth).toBe(24)
|
||||
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'ffprobe',
|
||||
expect.any(Array),
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
|
||||
it('should construct correct ffmpeg arguments for audio transcoding', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(child_process.execFile as any).mockImplementation((file: string, args: string[], cb: any) => {
|
||||
cb(null, { stdout: '', stderr: '' })
|
||||
})
|
||||
|
||||
await transcodeService.transcodeAudio({
|
||||
inputFile: 'input.wav',
|
||||
outputFile: 'output.mp4',
|
||||
bitrate: '128k',
|
||||
})
|
||||
|
||||
expect(child_process.execFile).toHaveBeenCalledWith(
|
||||
'ffmpeg',
|
||||
[
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'warning',
|
||||
'-i',
|
||||
'input.wav',
|
||||
'-vn',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
'128k',
|
||||
'-ac',
|
||||
'2',
|
||||
'output.mp4',
|
||||
],
|
||||
expect.any(Function),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -292,6 +292,67 @@ export class TranscodeService {
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
|
||||
async getAudioInfo(inputFile: string): Promise<MediaMetadata> {
|
||||
const { stdout } = await execFileAsync('ffprobe', [
|
||||
'-v',
|
||||
'quiet',
|
||||
'-print_format',
|
||||
'json',
|
||||
'-show_format',
|
||||
'-show_streams',
|
||||
inputFile,
|
||||
])
|
||||
const info = JSON.parse(stdout)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const audioStream = info.streams.find((s: any) => s.codec_type === 'audio')
|
||||
|
||||
if (!audioStream) {
|
||||
throw new Error('No audio stream found')
|
||||
}
|
||||
|
||||
const duration = parseFloat(info.format.duration)
|
||||
|
||||
return {
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
duration: isNaN(duration) ? 0 : duration,
|
||||
bitRate: parseFloat(info.format.bit_rate) || 0,
|
||||
frameRate: 0,
|
||||
totalFrames: 0,
|
||||
startTimecode: undefined,
|
||||
hasAudio: true,
|
||||
videoCodec: undefined,
|
||||
audioCodec: this.resolveCodecName(audioStream),
|
||||
audioChannels: audioStream?.channels,
|
||||
audioSampleRate: this.safeParseInt(audioStream?.sample_rate),
|
||||
audioBitDepth:
|
||||
this.safeParseInt(audioStream?.bits_per_raw_sample) ??
|
||||
this.safeParseInt(audioStream?.bits_per_sample),
|
||||
mimeType: '',
|
||||
}
|
||||
}
|
||||
|
||||
async transcodeAudio(params: {
|
||||
inputFile: string
|
||||
outputFile: string
|
||||
bitrate?: string
|
||||
}): Promise<void> {
|
||||
const bitrate = params.bitrate || '128k'
|
||||
const args = [
|
||||
'-i',
|
||||
params.inputFile,
|
||||
'-vn',
|
||||
'-c:a',
|
||||
'aac',
|
||||
'-b:a',
|
||||
bitrate,
|
||||
'-ac',
|
||||
'2',
|
||||
params.outputFile,
|
||||
]
|
||||
await execFileAsync('ffmpeg', ['-y', '-loglevel', 'warning', ...args])
|
||||
}
|
||||
|
||||
async extractVideoFrames(params: ExtractVideoFramesParams): Promise<string[]> {
|
||||
if (params.isImage) {
|
||||
const outputFile = path.join(params.outputDir, '1.webp')
|
||||
|
||||
@@ -307,7 +307,8 @@ export class UploadService {
|
||||
|
||||
const isVideo = asset.mediaType?.startsWith('video/')
|
||||
const isImage = asset.mediaType?.startsWith('image/')
|
||||
if (!isVideo && !isImage) {
|
||||
const isAudio = asset.mediaType?.startsWith('audio/')
|
||||
if (!isVideo && !isImage && !isAudio) {
|
||||
await tx.asset.update({
|
||||
where: { id: asset.id },
|
||||
data: { status: AssetStatus.processed },
|
||||
@@ -322,6 +323,9 @@ export class UploadService {
|
||||
.withSprite()
|
||||
.withPoster()
|
||||
.submit()
|
||||
} else if (isAudio) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new VideoTranscoder(tx as any, asset.id, team.id, projectId).submit()
|
||||
} else if (isImage) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await new ImageTranscoder(tx as any, asset.id, team.id, projectId).withThumbnail().submit()
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
overlayAnnotationsActivity,
|
||||
takeScreenshotsActivity,
|
||||
transcodeVideoActivity,
|
||||
transcodeAudioActivity,
|
||||
updateAssetMediaActivity,
|
||||
downloadMediaToTmpActivity,
|
||||
transcodeImageActivity,
|
||||
@@ -36,8 +37,10 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
|
||||
vi.mock('@shumai/core/src/transcode/transcode', () => ({
|
||||
transcodeService: {
|
||||
getVideoInfo: vi.fn(),
|
||||
getAudioInfo: vi.fn(),
|
||||
getImageInfo: vi.fn(),
|
||||
transcodeVideo: vi.fn(),
|
||||
transcodeAudio: vi.fn(),
|
||||
transcodeImage: vi.fn(),
|
||||
generateSprite: vi.fn(),
|
||||
createTempDir: vi.fn().mockReturnValue('/tmp'),
|
||||
@@ -143,20 +146,49 @@ describe('Transcode Activities', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should set file_type to audio for audio files', async () => {
|
||||
it('should set file_type to audio for audio files and extract audio metadata', async () => {
|
||||
const asset = await prisma.asset.create({
|
||||
data: { name: 'a.mp3', type: 'file', status: 'uploaded' },
|
||||
})
|
||||
vi.mocked(transcodeService.getAudioInfo).mockResolvedValue({
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
duration: 45.5,
|
||||
bitRate: 128000,
|
||||
frameRate: 0,
|
||||
totalFrames: 0,
|
||||
hasAudio: true,
|
||||
audioCodec: 'mp3',
|
||||
audioChannels: 2,
|
||||
audioSampleRate: 44100,
|
||||
audioBitDepth: 16,
|
||||
mimeType: 'audio/mpeg',
|
||||
})
|
||||
|
||||
await getMediaInfoActivity({
|
||||
const result = await getMediaInfoActivity({
|
||||
assetId: asset.id,
|
||||
filePath: '/tmp/a.mp3',
|
||||
mediaType: 'audio/mpeg',
|
||||
})
|
||||
|
||||
expect(transcodeService.getAudioInfo).toHaveBeenCalledWith('/tmp/a.mp3')
|
||||
expect(result.duration).toBe(45.5)
|
||||
expect(result.metadata?.duration).toBe(45.5)
|
||||
expect(result.metadata?.audioCodec).toBe('mp3')
|
||||
expect(result.metadata?.audioChannels).toBe(2)
|
||||
expect(result.metadata?.audioSampleRate).toBe(44100)
|
||||
expect(result.metadata?.audioBitDepth).toBe(16)
|
||||
|
||||
expect(metadataService.updateAssetMetadata).toHaveBeenCalledWith(
|
||||
asset.id,
|
||||
expect.arrayContaining([{ key: 'file_type', value: 'audio' }]),
|
||||
expect.arrayContaining([
|
||||
{ key: 'file_type', value: 'audio' },
|
||||
{ key: 'duration', value: 45.5 },
|
||||
{ key: 'audio_codec', value: 'mp3' },
|
||||
{ key: 'audio_channels', value: 2 },
|
||||
{ key: 'audio_sample_rate', value: 44100 },
|
||||
{ key: 'audio_bit_depth', value: 16 },
|
||||
]),
|
||||
true,
|
||||
)
|
||||
})
|
||||
@@ -437,5 +469,53 @@ describe('Transcode Activities', () => {
|
||||
await deleteS3ObjectActivity({ key: 'files/test-key.mp4' })
|
||||
expect(s3Service.deleteObject).toHaveBeenCalledWith('shumai', 'files/test-key.mp4')
|
||||
})
|
||||
|
||||
it('should transcode audio file and upload proxy to S3', async () => {
|
||||
vi.mocked(s3Service.headObject).mockRejectedValue(new Error('Not found'))
|
||||
vi.mocked(transcodeService.transcodeAudio).mockResolvedValue()
|
||||
|
||||
const res = await transcodeAudioActivity({
|
||||
assetKey: 'files/proj-123/a.wav',
|
||||
filePath: '/tmp/a.wav',
|
||||
})
|
||||
|
||||
expect(transcodeService.transcodeAudio).toHaveBeenCalledWith({
|
||||
inputFile: '/tmp/a.wav',
|
||||
outputFile: expect.stringContaining('a-audio-proxy.mp4'),
|
||||
bitrate: '128k',
|
||||
})
|
||||
|
||||
expect(s3Service.putObject).toHaveBeenCalledWith(
|
||||
'shumai',
|
||||
'files/proj-123/a-audio-proxy.mp4',
|
||||
expect.any(Buffer),
|
||||
expect.any(Number),
|
||||
'video/mp4',
|
||||
)
|
||||
|
||||
expect(res).toEqual({
|
||||
width: 0,
|
||||
height: 0,
|
||||
key: 'files/proj-123/a-audio-proxy.mp4',
|
||||
})
|
||||
})
|
||||
|
||||
it('should return existing audio transcode if already exists in S3', async () => {
|
||||
vi.mocked(s3Service.headObject).mockResolvedValue(
|
||||
{} as Awaited<ReturnType<typeof s3Service.headObject>>,
|
||||
)
|
||||
|
||||
const res = await transcodeAudioActivity({
|
||||
assetKey: 'files/proj-123/a.wav',
|
||||
filePath: '/tmp/a.wav',
|
||||
})
|
||||
|
||||
expect(transcodeService.transcodeAudio).not.toHaveBeenCalled()
|
||||
expect(res).toEqual({
|
||||
width: 0,
|
||||
height: 0,
|
||||
key: 'files/proj-123/a-audio-proxy.mp4',
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -116,6 +116,36 @@ export async function getMediaInfoActivity(params: {
|
||||
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
|
||||
if (info.audioBitDepth !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
|
||||
} else if (isAudio) {
|
||||
const info = await transcodeService.getAudioInfo(params.filePath)
|
||||
mediaInfo.duration = info.duration
|
||||
mediaInfo.metadata = {
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
duration: info.duration,
|
||||
bitRate: info.bitRate,
|
||||
frameRate: 0,
|
||||
totalFrames: 0,
|
||||
startTimecode: '00:00:00:00',
|
||||
hasAudio: true,
|
||||
videoCodec: undefined,
|
||||
audioCodec: info.audioCodec,
|
||||
audioChannels: info.audioChannels,
|
||||
audioSampleRate: info.audioSampleRate,
|
||||
audioBitDepth: info.audioBitDepth,
|
||||
format: {},
|
||||
}
|
||||
metadataUpdates.push(
|
||||
{ key: 'duration', value: info.duration },
|
||||
{ key: 'bitRate', value: info.bitRate / 1000 },
|
||||
)
|
||||
if (info.audioCodec) metadataUpdates.push({ key: 'audio_codec', value: info.audioCodec })
|
||||
if (info.audioChannels !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_channels', value: info.audioChannels })
|
||||
if (info.audioSampleRate !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_sample_rate', value: info.audioSampleRate })
|
||||
if (info.audioBitDepth !== undefined)
|
||||
metadataUpdates.push({ key: 'audio_bit_depth', value: info.audioBitDepth })
|
||||
} else if (isImage) {
|
||||
const info = await transcodeService.getImageInfo(params.filePath)
|
||||
mediaInfo.metadata = {
|
||||
@@ -230,6 +260,64 @@ export async function transcodeVideoActivity(
|
||||
}
|
||||
}
|
||||
|
||||
export interface AudioActivityParams {
|
||||
assetKey: string
|
||||
filePath: string
|
||||
}
|
||||
|
||||
export async function transcodeAudioActivity(
|
||||
params: AudioActivityParams,
|
||||
): Promise<PrismaJson.VideoTranscode> {
|
||||
const bucket = process.env.S3_BUCKET || 'shumai'
|
||||
const stem = stemFromKey(params.assetKey)
|
||||
const key = path.posix.join(path.posix.dirname(params.assetKey), `${stem}-audio-proxy.mp4`)
|
||||
|
||||
try {
|
||||
await s3Service.headObject(bucket, key)
|
||||
return { width: 0, height: 0, key }
|
||||
} catch {
|
||||
// Not found
|
||||
}
|
||||
|
||||
const tmpDir = path.dirname(params.filePath)
|
||||
const outputFile = path.join(tmpDir, `${stem}-audio-proxy.mp4`)
|
||||
|
||||
try {
|
||||
await transcodeService.transcodeAudio({
|
||||
inputFile: params.filePath,
|
||||
outputFile,
|
||||
bitrate: '128k',
|
||||
})
|
||||
|
||||
const buffer = fs.readFileSync(outputFile)
|
||||
await s3Service.putObject(bucket, key, buffer, buffer.length, 'video/mp4')
|
||||
|
||||
return { width: 0, height: 0, key }
|
||||
} catch (err) {
|
||||
const { code, message } = getErrorDetails(err)
|
||||
const lowerMsg = message.toLowerCase()
|
||||
|
||||
if (
|
||||
code === 'ENOENT' ||
|
||||
lowerMsg.includes('enoent') ||
|
||||
lowerMsg.includes('ffmpeg') ||
|
||||
lowerMsg.includes('ffprobe') ||
|
||||
lowerMsg.includes('spawn') ||
|
||||
lowerMsg.includes('format') ||
|
||||
lowerMsg.includes('no video stream found')
|
||||
) {
|
||||
throw ApplicationFailure.create({
|
||||
message: `Audio transcoding failed: ${message}`,
|
||||
nonRetryable: true,
|
||||
cause: err instanceof Error ? err : undefined,
|
||||
})
|
||||
}
|
||||
throw err
|
||||
} finally {
|
||||
if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
|
||||
}
|
||||
}
|
||||
|
||||
export interface ImageActivityParams {
|
||||
assetKey: string
|
||||
filePath: string
|
||||
|
||||
@@ -25,6 +25,7 @@ describe('Transcode Workflow', () => {
|
||||
getAssetActivity: Object.assign(vi.fn(), { _activityName: 'getAssetActivity' }),
|
||||
getMediaInfoActivity: Object.assign(vi.fn(), { _activityName: 'getMediaInfoActivity' }),
|
||||
transcodeVideoActivity: Object.assign(vi.fn(), { _activityName: 'transcodeVideoActivity' }),
|
||||
transcodeAudioActivity: Object.assign(vi.fn(), { _activityName: 'transcodeAudioActivity' }),
|
||||
transcodeImageActivity: Object.assign(vi.fn(), { _activityName: 'transcodeImageActivity' }),
|
||||
generateSpriteActivity: Object.assign(vi.fn(), { _activityName: 'generateSpriteActivity' }),
|
||||
updateAssetMediaActivity: Object.assign(vi.fn(), {
|
||||
@@ -388,4 +389,87 @@ describe('Transcode Workflow', () => {
|
||||
output: { key: 'files/asset-image/annotations/ann1.webp' },
|
||||
})
|
||||
})
|
||||
|
||||
it('should run audio transcoding workflow successfully', async () => {
|
||||
const task: WorkflowTask = {
|
||||
id: 'task-audio',
|
||||
assetId: 'asset-audio',
|
||||
type: WorkflowTaskType.transcode,
|
||||
status: WorkflowTaskStatus.pending,
|
||||
output: null,
|
||||
payload: {
|
||||
projectId: 'proj-1',
|
||||
transcode: {},
|
||||
},
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
heartbeat: null,
|
||||
teamId: 'team-1',
|
||||
projectId: 'proj-1',
|
||||
uid: 'task-uid-2',
|
||||
model: null,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
}
|
||||
|
||||
mockActivities.getAssetActivity.mockResolvedValue({
|
||||
id: 'asset-audio',
|
||||
storageKey: { key: 'audio.wav' },
|
||||
mediaType: 'audio/wav',
|
||||
})
|
||||
|
||||
mockActivities.getMediaInfoActivity.mockResolvedValue({
|
||||
mimeType: 'audio/wav',
|
||||
metadata: {
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
duration: 30,
|
||||
bitRate: 1411200,
|
||||
frameRate: 0,
|
||||
totalFrames: 0,
|
||||
hasAudio: true,
|
||||
audioCodec: 'pcm_s16le',
|
||||
audioChannels: 2,
|
||||
},
|
||||
original: {
|
||||
key: 'audio.wav',
|
||||
downloadUrl: '',
|
||||
filesizeInBytes: 0,
|
||||
codec: '',
|
||||
},
|
||||
videoTranscodes: [],
|
||||
})
|
||||
|
||||
mockActivities.transcodeAudioActivity.mockResolvedValue({
|
||||
width: 0,
|
||||
height: 0,
|
||||
key: 'files/asset-audio/audio-audio-proxy.mp4',
|
||||
})
|
||||
|
||||
await transcodeMedia(task)
|
||||
|
||||
expect(mockActivities.transcodeAudioActivity).toHaveBeenCalledWith({
|
||||
assetKey: 'audio.wav',
|
||||
filePath: '/tmp/video.mp4',
|
||||
})
|
||||
|
||||
expect(mockActivities.updateAssetMediaActivity).toHaveBeenCalledWith({
|
||||
assetId: 'asset-audio',
|
||||
mediaInfo: expect.objectContaining({
|
||||
mimeType: 'audio/wav',
|
||||
videoTranscodes: [
|
||||
{
|
||||
width: 0,
|
||||
height: 0,
|
||||
key: 'files/asset-audio/audio-audio-proxy.mp4',
|
||||
},
|
||||
],
|
||||
}),
|
||||
})
|
||||
|
||||
expect(mockActivities.updateTaskStatusActivity).toHaveBeenCalledWith({
|
||||
taskId: 'task-audio',
|
||||
status: WorkflowTaskStatus.completed,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -10,6 +10,7 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
getAssetActivity,
|
||||
getMediaInfoActivity,
|
||||
transcodeVideoActivity,
|
||||
transcodeAudioActivity,
|
||||
transcodeImageActivity,
|
||||
generateSpriteActivity,
|
||||
updateAssetMediaActivity,
|
||||
@@ -181,6 +182,16 @@ export async function transcodeMedia(task: WorkflowTask): Promise<void> {
|
||||
})
|
||||
}
|
||||
|
||||
// 6.5 Audio Transcoding
|
||||
const isAudio = mimeType.startsWith('audio/')
|
||||
if (isAudio && metadata) {
|
||||
const audioTranscode = await executeActivity(workerQueue, transcodeAudioActivity, {
|
||||
assetKey: key,
|
||||
filePath,
|
||||
})
|
||||
mediaInfo.videoTranscodes.push(audioTranscode)
|
||||
}
|
||||
|
||||
// 7. Image Transcoding
|
||||
const isImage = mimeType.startsWith('image/')
|
||||
const isPsd = isMimePsd(mimeType)
|
||||
|
||||
@@ -34,6 +34,7 @@ interface BreadcrumbNavProps {
|
||||
name?: string
|
||||
type: 'file' | 'folder'
|
||||
version?: number
|
||||
mediaType?: string | null
|
||||
}
|
||||
isRootFolder: boolean
|
||||
displayStyle?: 'card' | 'list'
|
||||
@@ -135,8 +136,11 @@ export function BreadcrumbNav({
|
||||
}
|
||||
}
|
||||
|
||||
const isAudio = currentAsset.mediaType?.startsWith('audio/')
|
||||
const hasVideoTranscodes =
|
||||
downloadInfo?.videoTranscodes && downloadInfo.videoTranscodes.filter((t) => !t.isRaw).length > 0
|
||||
!isAudio &&
|
||||
downloadInfo?.videoTranscodes &&
|
||||
downloadInfo.videoTranscodes.filter((t) => !t.isRaw).length > 0
|
||||
|
||||
const breadcrumbs: { name: string; path?: string; id?: string; isMuted?: boolean }[] = isPublic
|
||||
? [
|
||||
|
||||
@@ -3,7 +3,7 @@ import { client } from '@/ui/api/client'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
|
||||
import { useDraggable } from '@dnd-kit/react'
|
||||
import { File, Folder, MoreVertical } from 'lucide-react'
|
||||
import { File, Folder, MoreVertical, AudioLines } from 'lucide-react'
|
||||
import React, { useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { cn } from '@/ui/lib/utils'
|
||||
import type { DragState } from '../dnd-types'
|
||||
@@ -214,6 +214,9 @@ export function FileListItem({
|
||||
alt=""
|
||||
className="w-full h-full object-cover"
|
||||
/>
|
||||
) : displayItem.preview?.mediaType?.startsWith('audio/') ||
|
||||
displayItem.mediaType?.startsWith('audio/') ? (
|
||||
<AudioLines className="h-4 w-4 text-muted-foreground" />
|
||||
) : displayItem.type === 'folder' ? (
|
||||
<Folder className="h-4 w-4 text-primary" />
|
||||
) : (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { File, Folder } from 'lucide-react'
|
||||
import { File, Folder, AudioLines } from 'lucide-react'
|
||||
import { SpriteScrubber } from '../sprite-scrubber'
|
||||
import { formatTime } from '../viewers/video/utils'
|
||||
|
||||
@@ -63,6 +63,15 @@ export const FilePreview = ({ item, showDuration = false }: FilePreviewProps) =>
|
||||
)
|
||||
}
|
||||
|
||||
const isAudio = item.preview?.mediaType?.startsWith('audio/')
|
||||
if (isAudio) {
|
||||
return (
|
||||
<div className="w-full h-full flex items-center justify-center bg-muted">
|
||||
<AudioLines className="w-8 h-8 text-muted-foreground" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Assumption: folders in latestChildren have no media type.
|
||||
if (!item.type) {
|
||||
return (
|
||||
|
||||
@@ -2,7 +2,7 @@ import { client } from '@/ui/api/client'
|
||||
import { cn } from '@/ui/lib/utils'
|
||||
import type { AssetInfo, AssetInfoPaginatedList } from '@shumai/dtos'
|
||||
import { Link } from '@tanstack/react-router'
|
||||
import { File, Loader2 } from 'lucide-react'
|
||||
import { File, Loader2, AudioLines } from 'lucide-react'
|
||||
import type { FC } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { useInView } from 'react-intersection-observer'
|
||||
@@ -22,6 +22,16 @@ export const CarouselFilePreview = ({ item }: { item: AssetInfo }) => {
|
||||
)
|
||||
}
|
||||
|
||||
const isAudio =
|
||||
item.preview?.mediaType?.startsWith('audio/') || item.mediaType?.startsWith('audio/')
|
||||
if (isAudio) {
|
||||
return (
|
||||
<div className="w-full h-full flex items-center justify-center bg-muted">
|
||||
<AudioLines className="w-5 h-5 text-muted-foreground" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-full h-full flex items-center justify-center bg-muted">
|
||||
<File className="w-5 h-5 text-muted-foreground" />
|
||||
|
||||
@@ -412,6 +412,7 @@ export function PublicShareManager({
|
||||
? (stackFileData.versionStack.versions.find((v) => v.id === activeFileId)?.version ??
|
||||
stackFileData.versionStack.versions.length)
|
||||
: undefined,
|
||||
mediaType: viewingFileData?.mediaType || null,
|
||||
},
|
||||
isRootFolder: currentFolderId === shareInfo.rootFolderId && !viewingFileId,
|
||||
isPublic: true,
|
||||
|
||||
@@ -4,7 +4,7 @@ import { cn } from '@/ui/lib/utils'
|
||||
import { useAnnotationStore } from '@/ui/stores/annotation-store'
|
||||
import type { Annotation } from '@/ui/types'
|
||||
import type { AssetInfo } from '@shumai/dtos'
|
||||
import { Play } from 'lucide-react'
|
||||
import { Play, AudioLines } from 'lucide-react'
|
||||
import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from 'react'
|
||||
import videojs from 'video.js'
|
||||
import type Player from 'video.js/dist/types/player'
|
||||
@@ -138,8 +138,8 @@ export const CompareVideoPane = forwardRef<ComparePaneHandle, CompareVideoPanePr
|
||||
return () => observer.disconnect()
|
||||
}, [])
|
||||
|
||||
const vidW = metadata?.originalWidth ?? 1920
|
||||
const vidH = metadata?.originalHeight ?? 1080
|
||||
const vidW = metadata?.originalWidth || 1920
|
||||
const vidH = metadata?.originalHeight || 1080
|
||||
|
||||
// Auto-fit zoom
|
||||
useEffect(() => {
|
||||
@@ -439,22 +439,34 @@ export const CompareVideoPane = forwardRef<ComparePaneHandle, CompareVideoPanePr
|
||||
>
|
||||
<div ref={videoContainerRef} className="absolute inset-0 z-[-1]" />
|
||||
|
||||
{videoHtmlEl && containerSize.width > 0 && (
|
||||
<DrawingCanvas
|
||||
width={containerSize.width}
|
||||
height={containerSize.height}
|
||||
mediaDimensions={{ width: vidW, height: vidH }}
|
||||
videoElement={videoHtmlEl}
|
||||
annotations={displayAnnotations}
|
||||
scale={scale}
|
||||
offset={{ x: panX, y: panY }}
|
||||
className="absolute inset-0"
|
||||
onClick={handleAreaClick}
|
||||
isDrawing={isActive && isDrawing}
|
||||
currentTool={currentTool}
|
||||
currentColor={currentColor}
|
||||
onAddAnnotation={isActive ? addAnnotation : undefined}
|
||||
/>
|
||||
{file.mediaType?.startsWith('audio/') ? (
|
||||
<div className="flex flex-col items-center justify-center text-muted-foreground w-full h-full pointer-events-none select-none">
|
||||
<AudioLines
|
||||
className={cn(
|
||||
'w-16 h-16 text-foreground/75 transition-transform duration-500',
|
||||
isPlaying ? 'animate-pulse scale-110 text-primary' : '',
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
videoHtmlEl &&
|
||||
containerSize.width > 0 && (
|
||||
<DrawingCanvas
|
||||
width={containerSize.width}
|
||||
height={containerSize.height}
|
||||
mediaDimensions={{ width: vidW, height: vidH }}
|
||||
videoElement={videoHtmlEl}
|
||||
annotations={displayAnnotations}
|
||||
scale={scale}
|
||||
offset={{ x: panX, y: panY }}
|
||||
className="absolute inset-0"
|
||||
onClick={handleAreaClick}
|
||||
isDrawing={isActive && isDrawing}
|
||||
currentTool={currentTool}
|
||||
currentColor={currentColor}
|
||||
onAddAnnotation={isActive ? addAnnotation : undefined}
|
||||
/>
|
||||
)
|
||||
)}
|
||||
|
||||
{!isPlaying && !isDrawing && (
|
||||
|
||||
@@ -5,7 +5,7 @@ import { CompareVideoPane } from './compare-video-pane'
|
||||
export const videoTypeDefinition: FileTypeDefinition = {
|
||||
id: 'video',
|
||||
name: 'Video',
|
||||
match: (file) => !!file.mediaType?.startsWith('video/'),
|
||||
match: (file) => !!file.mediaType?.startsWith('video/') || !!file.mediaType?.startsWith('audio/'),
|
||||
viewer: VideoViewer,
|
||||
comparePane: CompareVideoPane,
|
||||
commentsConfig: {
|
||||
|
||||
@@ -113,6 +113,7 @@ export const VideoControlBar: React.FC<ControlBarProps> = ({
|
||||
// fullscreen). Compare keeps it in-flow. Colors stay theme-aware either way.
|
||||
const overlay = state.isFullScreen && floatOverlayInFullScreen
|
||||
|
||||
const isAudio = data.mediaType?.startsWith('audio/')
|
||||
const previewResolutions = resolutions.filter((r) => !r.isRaw)
|
||||
|
||||
const startTimecode = data.media?.metadata?.startTimecode
|
||||
@@ -304,60 +305,73 @@ export const VideoControlBar: React.FC<ControlBarProps> = ({
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
|
||||
{/* Resolution Selector */}
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button className="flex items-center gap-1 rounded border border-border px-2 py-0.5 text-sm font-semibold hover:bg-muted transition-colors">
|
||||
<Settings className="h-3.5 w-3.5" />
|
||||
<span>{state.currentResolution}</span>
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
{/* Settings / Resolution */}
|
||||
{!isAudio && previewResolutions.length > 0 && (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button className="flex items-center gap-1 rounded border border-border px-2 py-0.5 text-sm font-semibold hover:bg-muted transition-colors">
|
||||
<Settings className="h-3.5 w-3.5" />
|
||||
<span>{state.currentResolution}</span>
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
|
||||
<DropdownMenuContent>
|
||||
<DropdownMenuLabel>Quality</DropdownMenuLabel>
|
||||
{previewResolutions.map((res) => (
|
||||
<DropdownMenuItem
|
||||
key={res.resolution}
|
||||
onClick={() => changeResolution(res)}
|
||||
className={cn(
|
||||
'flex w-full items-center justify-between',
|
||||
state.currentResolution === res.resolution
|
||||
? 'text-primary font-medium'
|
||||
: 'text-foreground',
|
||||
)}
|
||||
>
|
||||
<span>{res.resolution}</span>
|
||||
<span className="pl-5 text-xs text-muted-foreground">
|
||||
{res.width}x{res.height}
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
<DropdownMenuContent>
|
||||
<DropdownMenuLabel>Quality</DropdownMenuLabel>
|
||||
{previewResolutions.map((res) => (
|
||||
<DropdownMenuItem
|
||||
key={res.resolution}
|
||||
onClick={() => changeResolution(res)}
|
||||
className={cn(
|
||||
'flex w-full items-center justify-between',
|
||||
state.currentResolution === res.resolution
|
||||
? 'text-primary font-medium'
|
||||
: 'text-foreground',
|
||||
)}
|
||||
>
|
||||
<span>{res.resolution}</span>
|
||||
<span className="pl-5 text-xs text-muted-foreground">
|
||||
{res.width}x{res.height}
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
|
||||
{/* Download */}
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button className="transition-colors hover:text-primary" title="Download">
|
||||
<Download className="h-5 w-5" />
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent>
|
||||
<DropdownMenuLabel>Download</DropdownMenuLabel>
|
||||
{resolutions.map((res) => (
|
||||
<DropdownMenuItem
|
||||
key={res.resolution}
|
||||
onClick={() => handleDownload(res.key ?? '')}
|
||||
className="flex w-full items-center justify-between px-3 py-2 text-left text-sm text-foreground"
|
||||
>
|
||||
<span>{res.resolution}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{res.isRaw ? data.name?.split('.').pop()?.toUpperCase() || 'RAW' : 'MP4'}
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
{isAudio ? (
|
||||
<button
|
||||
onClick={() => handleDownload(data.media?.original?.key || '')}
|
||||
className="transition-colors hover:text-primary"
|
||||
title="Download"
|
||||
disabled={!data.media?.original?.key}
|
||||
>
|
||||
<Download className="h-5 w-5" />
|
||||
</button>
|
||||
) : (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<button className="transition-colors hover:text-primary" title="Download">
|
||||
<Download className="h-5 w-5" />
|
||||
</button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent>
|
||||
<DropdownMenuLabel>Download</DropdownMenuLabel>
|
||||
{resolutions.map((res) => (
|
||||
<DropdownMenuItem
|
||||
key={res.resolution}
|
||||
onClick={() => handleDownload(res.key ?? '')}
|
||||
className="flex w-full items-center justify-between px-3 py-2 text-left text-sm text-foreground"
|
||||
>
|
||||
<span>{res.resolution}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{res.isRaw ? data.name?.split('.').pop()?.toUpperCase() || 'RAW' : 'MP4'}
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
))}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
|
||||
{/* Fullscreen */}
|
||||
<button onClick={toggleFullScreen} className="transition-colors hover:text-primary">
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { client } from '@/ui/api/client'
|
||||
import { cn } from '@/ui/lib/utils'
|
||||
import { Play } from 'lucide-react'
|
||||
import { Play, AudioLines } from 'lucide-react'
|
||||
import React, { useCallback, useEffect, useRef, useState, useImperativeHandle } from 'react'
|
||||
import videojs from 'video.js'
|
||||
import type Player from 'video.js/dist/types/player'
|
||||
@@ -96,8 +96,8 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
|
||||
// Fit to screen initial / responsive resize
|
||||
useEffect(() => {
|
||||
if (containerSize.width > 0 && containerSize.height > 0) {
|
||||
const vidW = data.media?.metadata?.originalWidth ?? 1920
|
||||
const vidH = data.media?.metadata?.originalHeight ?? 1080
|
||||
const vidW = data.media?.metadata?.originalWidth || 1920
|
||||
const vidH = data.media?.metadata?.originalHeight || 1080
|
||||
const scale = Math.min(containerSize.width / vidW, containerSize.height / vidH)
|
||||
if (!hasManuallyZoomed) {
|
||||
setZoom(scale)
|
||||
@@ -541,8 +541,8 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
|
||||
}, [])
|
||||
|
||||
// Calculate center pan
|
||||
const vidW = data.media?.metadata?.originalWidth ?? 1920
|
||||
const vidH = data.media?.metadata?.originalHeight ?? 1080
|
||||
const vidW = data.media?.metadata?.originalWidth || 1920
|
||||
const vidH = data.media?.metadata?.originalHeight || 1080
|
||||
|
||||
const scale = zoom
|
||||
const panX = (containerSize.width - vidW * scale) / 2
|
||||
@@ -584,27 +584,39 @@ const VideoViewer = React.forwardRef<MediaController, FileViewerProps>(
|
||||
{/* Hidden VideoJS container */}
|
||||
<div ref={videoContainerRef} className="absolute inset-0 z-[-1]" />
|
||||
|
||||
{/* Drawing Canvas (Visible) */}
|
||||
{videoHtmlEl && containerSize.width > 0 && (
|
||||
<DrawingCanvas
|
||||
width={containerSize.width}
|
||||
height={containerSize.height}
|
||||
mediaDimensions={{
|
||||
width: vidW,
|
||||
height: vidH,
|
||||
}}
|
||||
videoElement={videoHtmlEl}
|
||||
annotations={displayAnnotations}
|
||||
scale={scale}
|
||||
offset={{ x: panX, y: panY }}
|
||||
className="absolute inset-0"
|
||||
onClick={togglePlay}
|
||||
// Drawing Props
|
||||
isDrawing={isDrawing}
|
||||
currentTool={currentTool}
|
||||
currentColor={currentColor}
|
||||
onAddAnnotation={addAnnotation}
|
||||
/>
|
||||
{data.mediaType?.startsWith('audio/') ? (
|
||||
<div className="flex flex-col items-center justify-center text-muted-foreground w-full h-full pointer-events-none select-none">
|
||||
<AudioLines
|
||||
className={cn(
|
||||
'w-16 h-16 text-foreground/75 transition-transform duration-500',
|
||||
state.isPlaying ? 'animate-pulse scale-110 text-primary' : '',
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
/* Drawing Canvas (Visible) */
|
||||
videoHtmlEl &&
|
||||
containerSize.width > 0 && (
|
||||
<DrawingCanvas
|
||||
width={containerSize.width}
|
||||
height={containerSize.height}
|
||||
mediaDimensions={{
|
||||
width: vidW,
|
||||
height: vidH,
|
||||
}}
|
||||
videoElement={videoHtmlEl}
|
||||
annotations={displayAnnotations}
|
||||
scale={scale}
|
||||
offset={{ x: panX, y: panY }}
|
||||
className="absolute inset-0"
|
||||
onClick={togglePlay}
|
||||
// Drawing Props
|
||||
isDrawing={isDrawing}
|
||||
currentTool={currentTool}
|
||||
currentColor={currentColor}
|
||||
onAddAnnotation={addAnnotation}
|
||||
/>
|
||||
)
|
||||
)}
|
||||
|
||||
{/* Big Play Button Overlay (when paused) */}
|
||||
|
||||
@@ -155,6 +155,7 @@ function FileViewPage() {
|
||||
? (versionsDataList.find((v: { id: string }) => v.id === activeFileId)?.version ??
|
||||
versionsDataList.length)
|
||||
: undefined,
|
||||
mediaType: fileData.mediaType,
|
||||
},
|
||||
isRootFolder: false,
|
||||
fileId,
|
||||
|
||||
@@ -10,6 +10,7 @@ export interface TopNavProjectState {
|
||||
name?: string
|
||||
type: 'file' | 'folder'
|
||||
version?: number
|
||||
mediaType?: string | null
|
||||
}
|
||||
isRootFolder: boolean
|
||||
fileId?: string
|
||||
|
||||
Reference in New Issue
Block a user