diff --git a/src/api/file.test.ts b/src/api/file.test.ts index 82d73911..bdb3e551 100644 --- a/src/api/file.test.ts +++ b/src/api/file.test.ts @@ -157,10 +157,11 @@ describe('file api', () => { it('POST /files/:fileId/comments', async () => { vi.mocked(assetService.createComment).mockResolvedValue({ id: 'comment-id', - assetId: 'test-id', + assetId: 'file-id', message: 'hello', annotations: null, - creator: { id: 'user1', name: 'user-name' }, + second: null, + creator: { id: 'user-id', name: 'Test User' }, replies: [], attachments: [], mentions: [], @@ -207,10 +208,11 @@ describe('file api', () => { data: [ { id: 'comment-id', - assetId: 'test-id', + assetId: 'file-id', message: 'hello', annotations: null, - creator: { id: 'user1', name: 'user-name' }, + second: null, + creator: { id: 'user-id', name: 'Test User' }, replies: [], attachments: [], mentions: [], diff --git a/src/dtos/asset.ts b/src/dtos/asset.ts index cee55d07..3faa3771 100644 --- a/src/dtos/asset.ts +++ b/src/dtos/asset.ts @@ -189,6 +189,7 @@ export type CommentInfo = { message: string | null // eslint-disable-next-line @typescript-eslint/no-explicit-any annotations: any + second: number | null creator: UserInfo replies: CommentInfo[] attachments: AttachmentInfo[] @@ -204,6 +205,7 @@ export const commentInfoSchema: z.ZodType = z.object({ message: z.string().nullable(), annotations: z.any(), + second: z.number().nullable(), creator: userInfoSchema, replies: z.array(z.lazy(() => commentInfoSchema)), attachments: z.array(attachmentInfoSchema), diff --git a/src/services/asset/asset.test.ts b/src/services/asset/asset.test.ts index 8b9aaa57..b91e0fdb 100644 --- a/src/services/asset/asset.test.ts +++ b/src/services/asset/asset.test.ts @@ -664,6 +664,28 @@ describe('AssetService', () => { expect(list.data[0].replies.length).toBe(1) }) + it('can create and retrieve comments with video timestamp', async () => { + const { user, assets } = await setupBasicAssets() + + const comment = await assetService.createComment({ + assetId: assets.fileA1.id, + userId: user.id, + message: 'Interesting point at 5.5s', + second: 5.5, + attachmentIds: [], + }) + + expect(comment.second).toBe(5.5) + + const retrieved = await assetService.getComment(comment.id) + expect(retrieved.second).toBe(5.5) + + const list = await assetService.listComments(assets.fileA1.id, { + first: 10, + }) + expect(list.data[0].second).toBe(5.5) + }) + it('correctly handles AI comments with agent identity', async () => { const { assets, project } = await setupBasicAssets() diff --git a/src/services/asset/asset.ts b/src/services/asset/asset.ts index fb4e15d9..e1e66396 100644 --- a/src/services/asset/asset.ts +++ b/src/services/asset/asset.ts @@ -1690,6 +1690,7 @@ export class AssetService { assetId: c.assetId, message: c.message, annotations: c.annotation, + second: c.second, creator, replies, attachments, diff --git a/src/ui/components/chat/message-card.tsx b/src/ui/components/chat/message-card.tsx index 8b6b37d5..ae5b6245 100644 --- a/src/ui/components/chat/message-card.tsx +++ b/src/ui/components/chat/message-card.tsx @@ -10,13 +10,14 @@ import { DialogTitle, } from '@/ui/components/ui/dialog' import { Separator } from '@/ui/components/ui/separator' +import { formatTimeAgo } from '@/ui/lib/time' import { useQuery } from '@tanstack/react-query' import { Download, File, Terminal } from 'lucide-react' import React from 'react' -import { formatTimeAgo } from '@/ui/lib/time' import Markdown from 'react-markdown' import { Avatar, AvatarFallback } from '../ui/avatar' import { Skeleton } from '../ui/skeleton' +import { formatTimestamp } from '../viewers/utils' interface MessageCardProps { teamId?: string @@ -27,6 +28,9 @@ interface MessageCardProps { getUser: (id: string) => UserInfo onReply: (message: CommentInfo) => void onViewAttachment: (attachment: AttachmentInfo) => void + isSelected?: boolean + onSelect?: () => void + frameRate?: number rootParentId?: string } @@ -48,6 +52,9 @@ export const MessageCard: React.FC = ({ getUser, onReply, onViewAttachment, + isSelected, + onSelect, + frameRate, }) => { const isRunning = !!initialMessage.sessionId && @@ -133,9 +140,16 @@ export const MessageCard: React.FC = ({ } return ( -
+
{hasReplies && !isReply && ( -
+
)} {isReply && !isLastReply && ( @@ -161,21 +175,29 @@ export const MessageCard: React.FC = ({
- {showSkeleton ? ( -
- - - {loadingText} -
- ) : message.sessionId ? ( -
- {preprocessMarkdown(message.message!)} -
- ) : ( -
- {renderFormattedMessage(message.message!)} -
- )} +
+ {message.second !== null && message.second !== undefined && frameRate && ( +
+ {formatTimestamp(message.second, frameRate)} +
+ )} + + {showSkeleton ? ( +
+ + + {loadingText} +
+ ) : message.sessionId ? ( +
+ {preprocessMarkdown(message.message!)} +
+ ) : ( +
+ {renderFormattedMessage(message.message!)} +
+ )} +
{/* Attachments */} {message.attachments && message.attachments.length > 0 && ( diff --git a/src/ui/components/chat/message-input.tsx b/src/ui/components/chat/message-input.tsx index 014b64cc..20fdc124 100644 --- a/src/ui/components/chat/message-input.tsx +++ b/src/ui/components/chat/message-input.tsx @@ -1,13 +1,16 @@ -import { client } from '@/ui/api/client' -import type { CommentInfo } from '@/dtos/asset' -import type { UserInfo } from '@/dtos/team' +import type { CommentInfo, PostAttachmentRequest, PostAttachmentResponse } from '@/dtos/asset' import type { BotInfo } from '@/dtos/project' -import type { PostAttachmentRequest, PostAttachmentResponse } from '@/dtos/asset' +import type { UserInfo } from '@/dtos/team' +import { client } from '@/ui/api/client' +import { useAnnotationStore } from '@/ui/stores/annotation-store' +import type { Annotation } from '@/ui/types' +import { useMutation } from '@tanstack/react-query' import { ArrowLeft, ArrowUp, Brush, ChevronDown, + Clock, FileIcon, Minus, Paperclip, @@ -19,11 +22,9 @@ import { import React, { useCallback, useEffect, useRef, useState, type ChangeEvent } from 'react' import { Button } from '../ui/button' import { DrawAnnotation } from '../ui/icons' -import { ProgressCircle } from '../ui/progress-circle' import { Popover, PopoverContent, PopoverTrigger } from '../ui/popover' -import { useAnnotationStore } from '@/ui/stores/annotation-store' -import type { Annotation } from '@/ui/types' -import { useMutation } from '@tanstack/react-query' +import { ProgressCircle } from '../ui/progress-circle' +import { formatTimestamp } from '../viewers/utils' type UploadingFile = { id: string // A unique ID for the file, e.g., timestamp + name @@ -40,6 +41,7 @@ interface ChatInputProps { attachmentIds: string[], annotations?: Annotation[], replyToId?: string | null, + second?: number | null, ) => void replyingTo?: CommentInfo | null onCancelReply?: () => void @@ -48,6 +50,9 @@ interface ChatInputProps { initialText?: string hideAnnotationControl?: boolean disableMentions?: boolean + currentTime?: number + frameRate?: number + onTyping?: () => void } const PREDEFINED_COLORS = [ @@ -76,11 +81,15 @@ export const ChatInput = React.forwardRef( initialText = '', hideAnnotationControl = false, disableMentions = false, + currentTime, + frameRate, + onTyping, }, ref, ) => { const [uploadingFiles, setUploadingFiles] = useState([]) const [viewingFile, setViewingFile] = useState(null) + const [isTimestampEnabled, setIsTimestampEnabled] = useState(true) // Annotation Store const { @@ -190,6 +199,7 @@ export const ChatInput = React.forwardRef( const handleInput = () => { if (!editorRef.current) return setHasContent(!!editorRef.current.innerText.trim()) + onTyping?.() if (disableMentions) { setShowMentionList(false) @@ -395,7 +405,13 @@ export const ChatInput = React.forwardRef( // Allow sending if annotations exist, even if text is empty if (!processedText && successfulAttachmentIds.length === 0 && annotations.length === 0) return - onSendMessage(processedText, successfulAttachmentIds, annotations, replyingTo?.id) + onSendMessage( + processedText, + successfulAttachmentIds, + annotations, + replyingTo?.id, + isTimestampEnabled ? currentTime : undefined, + ) // Reset editor editorRef.current.innerHTML = '' @@ -566,7 +582,21 @@ export const ChatInput = React.forwardRef(
)} -
+
editorRef.current?.focus()} + className="w-full px-4 pt-1 pb-1 cursor-text flow-root max-h-60 overflow-y-auto" + > + {isTimestampEnabled && currentTime !== undefined && frameRate !== undefined && ( +
{ + e.stopPropagation() + editorRef.current?.focus() + }} + className="float-left select-none bg-blue-100 dark:bg-blue-900/40 text-blue-600 dark:text-blue-300 px-2.5 py-0.5 mt-3 mr-2 rounded-sm text-xs font-mono font-bold flex items-center gap-1 cursor-text" + > + {formatTimestamp(currentTime, frameRate)} +
+ )}
( data-placeholder={ isDrawing ? 'Drawing active...' : replyingTo ? 'Reply...' : 'Message...' } - className="w-full bg-transparent border-none focus:ring-0 resize-none max-h-60 min-h-[40px] leading-relaxed py-2 focus:outline-none overflow-y-auto" + className={`bg-transparent border-none focus:ring-0 resize-none min-h-[40px] leading-relaxed py-2 focus:outline-none block ${isTimestampEnabled && currentTime !== undefined && frameRate !== undefined && !hasContent ? 'pl-[120px]' : ''}`} />
@@ -679,6 +709,17 @@ export const ChatInput = React.forwardRef( multiple onChange={handleFileSelect} /> + {currentTime !== undefined && frameRate !== undefined && ( + + )}
)} - + Comments @@ -328,11 +340,11 @@ export function FileViewerRightSidebar({ Fields - - + +
{comments.map((comment) => ( -
onCommentSelect?.(comment)}> +
{ + onCommentSelect?.(comment) + }} /> {comment.replies?.map((reply, index) => ( - +
+ { + onCommentSelect?.(reply) + }} + /> +
))}
))} @@ -375,6 +406,13 @@ export function FileViewerRightSidebar({ bots={enabledBots} hideAnnotationControl={hideAnnotationControl} disableMentions={isPublic} + currentTime={file?.mediaType?.startsWith('video/') ? currentTime : undefined} + frameRate={ + file?.mediaType?.startsWith('video/') + ? file?.media?.metadata?.frameRate || 30 + : undefined + } + onTyping={onTyping} />
diff --git a/src/ui/components/file-viewer.tsx b/src/ui/components/file-viewer.tsx index bea8cc0a..9d70bbc0 100644 --- a/src/ui/components/file-viewer.tsx +++ b/src/ui/components/file-viewer.tsx @@ -16,10 +16,18 @@ type FileViewerProps = { videoRef?: RefObject onPlay?: () => void onPause?: () => void + onTimeUpdate?: (time: number) => void annotations?: Annotation[] } -export function FileViewer({ file, videoRef, onPlay, onPause, annotations }: FileViewerProps) { +export function FileViewer({ + file, + videoRef, + onPlay, + onPause, + onTimeUpdate, + annotations, +}: FileViewerProps) { const { width: screenWidth } = useScreenSize() const isImage = file.mediaType?.startsWith('image/') const isVideo = file.mediaType?.startsWith('video/') @@ -186,6 +194,7 @@ export function FileViewer({ file, videoRef, onPlay, onPause, annotations }: Fil playerRef={videoRef} onPlay={onPlay} onPause={onPause} + onTimeUpdate={onTimeUpdate} annotations={displayAnnotations} /> )} diff --git a/src/ui/components/public-share-manager.tsx b/src/ui/components/public-share-manager.tsx index e3f9f4ad..31d95d8d 100644 --- a/src/ui/components/public-share-manager.tsx +++ b/src/ui/components/public-share-manager.tsx @@ -89,6 +89,8 @@ export function PublicShareManager({ const [isRightSidebarCollapsed] = useState(false) const videoRef = useRef(null) const [annotations, setAnnotations] = useState([]) + const [currentTime, setCurrentTime] = useState(0) + const [selectedCommentId, setSelectedCommentId] = useState(null) const { data: foldersData, @@ -254,19 +256,29 @@ export function PublicShareManager({ } const handleCommentSelect = (comment: CommentInfo) => { + setSelectedCommentId(comment.id || null) const newAnnotations = comment.annotations as Annotation[] if (newAnnotations && newAnnotations.length > 0) { setAnnotations(newAnnotations) + } else { + setAnnotations([]) + } + + if (comment.second !== null && comment.second !== undefined) { + if (videoRef.current) { + videoRef.current.currentTime(comment.second) + videoRef.current.pause() + } + } else if (newAnnotations && newAnnotations.length > 0) { if (videoRef.current) { videoRef.current.pause() } - } else { - setAnnotations([]) } } const handlePlay = () => { setAnnotations([]) + setSelectedCommentId(null) } const { setProjectState, clearProjectState } = useTopNavStore() @@ -373,6 +385,7 @@ export function PublicShareManager({ file={viewingFileData} videoRef={videoRef} onPlay={handlePlay} + onTimeUpdate={setCurrentTime} annotations={annotations} /> )} @@ -430,6 +443,13 @@ export function PublicShareManager({ onCommentSelect={handleCommentSelect} isPublic={true} shareId={shareInfo.id} + currentTime={currentTime} + onTyping={() => { + if (videoRef.current) { + videoRef.current.pause() + } + }} + selectedCommentId={selectedCommentId} />
diff --git a/src/ui/components/viewers/utils.ts b/src/ui/components/viewers/utils.ts index 0c5a4593..b20f9dd4 100644 --- a/src/ui/components/viewers/utils.ts +++ b/src/ui/components/viewers/utils.ts @@ -10,6 +10,16 @@ export const formatTime = (seconds: number): string => { return `${mm}:${ss}` } +export const formatTimestamp = (seconds: number, fps: number): string => { + if (isNaN(seconds)) return '00:00:00:00' + const hh = Math.floor(seconds / 3600) + const mm = Math.floor((seconds % 3600) / 60) + const ss = Math.floor(seconds % 60) + const ff = Math.floor((seconds % 1) * fps) + + return `${hh.toString().padStart(2, '0')}:${mm.toString().padStart(2, '0')}:${ss.toString().padStart(2, '0')}:${ff.toString().padStart(2, '0')}` +} + export const formatFrame = (seconds: number, fps: number): number => { if (isNaN(seconds) || !fps) return 0 return Math.floor(seconds * fps) diff --git a/src/ui/components/viewers/video-player.tsx b/src/ui/components/viewers/video-player.tsx index aae83806..9e8c4aa7 100644 --- a/src/ui/components/viewers/video-player.tsx +++ b/src/ui/components/viewers/video-player.tsx @@ -51,6 +51,7 @@ interface VideoPlayerProps { playerRef?: React.RefObject onPlay?: () => void onPause?: () => void + onTimeUpdate?: (time: number) => void annotations?: Annotation[] } @@ -299,6 +300,7 @@ const VideoPlayer: React.FC = ({ playerRef: passedPlayerRef, onPlay, onPause, + onTimeUpdate, annotations, }) => { const localPlayerRef = useRef(null) @@ -494,6 +496,10 @@ const VideoPlayer: React.FC = ({ duration: duration || prev.duration, progress: duration > 0 ? (current / duration) * 100 : 0, })) + + if (onTimeUpdate) { + onTimeUpdate(current) + } }) player.on('volumechange', () => { diff --git a/src/ui/routes/__root.tsx b/src/ui/routes/__root.tsx index 0c3700be..b22b2295 100644 --- a/src/ui/routes/__root.tsx +++ b/src/ui/routes/__root.tsx @@ -8,10 +8,10 @@ import { useAuthStore } from '@/ui/stores/auth' import { useTeamContextStore } from '@/ui/stores/team-context' import { useUploadStore } from '@/ui/stores/upload' import { - createRootRouteWithContext, - Outlet, - useNavigate, - useRouterState, + createRootRouteWithContext, + Outlet, + useNavigate, + useRouterState, } from '@tanstack/react-router' import { useEffect } from 'react' diff --git a/src/ui/routes/projects/$projectId/files/$fileId.tsx b/src/ui/routes/projects/$projectId/files/$fileId.tsx index 2f928103..ea8997a9 100644 --- a/src/ui/routes/projects/$projectId/files/$fileId.tsx +++ b/src/ui/routes/projects/$projectId/files/$fileId.tsx @@ -34,6 +34,8 @@ function FileViewPage() { const [leftSidebarWidth, setLeftSidebarWidth] = useState(280) const [rightSidebarWidth, setRightSidebarWidth] = useState(360) const [annotations, setAnnotations] = useState([]) + const [currentTime, setCurrentTime] = useState(0) + const [selectedCommentId, setSelectedCommentId] = useState(null) const $patchMetadata = client.api.files[':fileId'].metadata.$patch const { mutate: patchMetadata } = useMutation< InferResponseType, @@ -194,19 +196,29 @@ function FileViewPage() { } const handleCommentSelect = (comment: CommentInfo) => { + setSelectedCommentId(comment.id || null) const newAnnotations = comment.annotations as Annotation[] if (newAnnotations && newAnnotations.length > 0) { setAnnotations(newAnnotations) + } else { + setAnnotations([]) + } + + if (comment.second !== null && comment.second !== undefined) { + if (videoRef.current) { + videoRef.current.currentTime(comment.second) + videoRef.current.pause() + } + } else if (newAnnotations && newAnnotations.length > 0) { if (videoRef.current) { videoRef.current.pause() } - } else { - setAnnotations([]) } } const handlePlay = () => { setAnnotations([]) + setSelectedCommentId(null) } return ( @@ -244,6 +256,7 @@ function FileViewPage() { file={fileData as unknown as AssetInfo} videoRef={videoRef} onPlay={handlePlay} + onTimeUpdate={setCurrentTime} annotations={annotations} />
@@ -251,18 +264,25 @@ function FileViewPage() { <> { - setRightSidebarWidth((prev) => Math.max(240, Math.min(600, prev - delta))) + setRightSidebarWidth((prev) => Math.max(300, Math.min(600, prev - delta))) }} className="hidden md:block" />
{ + if (videoRef.current) { + videoRef.current.pause() + } + }} + selectedCommentId={selectedCommentId} />