From 5947c3c9a73b724c010e71afd092b17bd312d464 Mon Sep 17 00:00:00 2001 From: yiling Date: Tue, 2 Jun 2026 16:06:24 +0800 Subject: [PATCH] refactor(workflow): eliminate worker-db package and simplify workflows --- AGENTS.md | 5 +- apps/web/package.json | 1 - apps/web/src/index.ts | 4 +- build.ts | 1 - bun.lock | 22 +- package.json | 3 + packages/agent/src/activities/agent.ts | 857 +++++++++++++- packages/agent/src/activities/ai.ts | 67 ++ .../agent/src/workflows/agent-autofill.ts | 33 +- packages/agent/src/workflows/agent-chat.ts | 32 +- .../agent/src/workflows/agent-embedding.ts | 22 +- .../agent/src/workflows/agent-tool-call.ts | 12 +- .../workflows/query-embedding-for-search.ts | 10 +- packages/transcode/src/workflows/transcode.ts | 15 +- packages/worker-db/package.json | 20 - packages/worker-db/src/activities/db.test.ts | 645 ----------- packages/worker-db/src/activities/db.ts | 1025 ----------------- packages/worker-db/src/index.ts | 8 - packages/worker-db/tsconfig.json | 7 - .../workflow-core/src/activities/task.test.ts | 2 +- packages/workflow-core/src/activities/task.ts | 95 +- .../workflow-core/src/temporal-executor.ts | 12 +- packages/workflow-core/src/workflow-utils.ts | 1 - packages/workflow-core/src/workflow.test.ts | 2 - packages/workflow-core/src/workflow.ts | 14 +- tsconfig.json | 3 +- 26 files changed, 1097 insertions(+), 1821 deletions(-) delete mode 100644 packages/worker-db/package.json delete mode 100644 packages/worker-db/src/activities/db.test.ts delete mode 100644 packages/worker-db/src/activities/db.ts delete mode 100644 packages/worker-db/src/index.ts delete mode 100644 packages/worker-db/tsconfig.json diff --git a/AGENTS.md b/AGENTS.md index c7fda0db..d32c2ac9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,7 +122,6 @@ The project is a monorepo managed by **Bun Workspaces**. It follows a strictly d - `@shumai/workflow-core`: Common workflow engine logic. - `@shumai/agent`: AI agent workflows and activities. - `@shumai/transcode`: Media processing workflows and activities. - - `@shumai/worker-db`: Shared database activities. ### Layered Communication Rules @@ -339,7 +338,7 @@ We use a custom workflow engine that supports both **Local** (polling-based) and ### Architecture 1. **Workflows**: Orchestrate multiple activities. They must be deterministic and compatible with Temporal's V8 isolate. Defined in worker packages (e.g., `@shumai/agent/src/workflows`, `@shumai/transcode/src/workflows`). -2. **Activities**: Perform the actual work (DB updates, API calls, media processing). Grouped by domain in worker packages (e.g., `@shumai/agent/src/activities`, `@shumai/transcode/src/activities`, `@shumai/worker-db/src/activities`). +2. **Activities**: Perform the actual work (DB updates, API calls, media processing). Grouped by domain in worker packages (e.g., `@shumai/agent/src/activities`, `@shumai/transcode/src/activities`). 3. **Executors**: - `LocalExecutor` (in `@shumai/workflow-core`): Polls the database for `pending` tasks and executes them directly. - `TemporalExecutor` (in `@shumai/workflow-core`): Submits tasks to a Temporal cluster. @@ -347,7 +346,7 @@ We use a custom workflow engine that supports both **Local** (polling-based) and ### Dynamic Registration -To avoid circular dependencies, workflows and activities are dynamically registered onto the `LocalExecutor` registry at bootstrap time via initializer functions (e.g., `initAgentWorkflows()`, `initTranscodeWorkflows()`, `initDbWorkflows()`). +To avoid circular dependencies, workflows and activities are dynamically registered onto the `LocalExecutor` registry at bootstrap time via initializer functions (e.g., `initAgentWorkflows()`, `initTranscodeWorkflows()`). ### Non-Retryable Error Handling diff --git a/apps/web/package.json b/apps/web/package.json index c8944846..c5bfd4e5 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,7 +11,6 @@ "@shumai/workflow-core": "workspace:*", "@shumai/agent": "workspace:*", "@shumai/transcode": "workspace:*", - "@shumai/worker-db": "workspace:*", "@shumai/webui": "workspace:*", "hono": "^4.12.14" } diff --git a/apps/web/src/index.ts b/apps/web/src/index.ts index 7f41c1d4..0b57405e 100644 --- a/apps/web/src/index.ts +++ b/apps/web/src/index.ts @@ -7,14 +7,12 @@ import { initAgentWorkflows } from '@shumai/agent' import { assetService } from '@shumai/core/src/asset/asset' import { metadataService } from '@shumai/core/src/metadata/metadata' import { initTranscodeWorkflows } from '@shumai/transcode' -import { initDbWorkflows } from '@shumai/worker-db' import { workflowService } from '@shumai/workflow-core' import { app } from '@shumai/api' // Initialize workflows and activities for local executor mode initAgentWorkflows() initTranscodeWorkflows() -initDbWorkflows() // Start services await metadataService.syncSystemFields().catch(console.error) @@ -31,7 +29,7 @@ if (process.env.WORKFLOW_EXECUTOR === 'temporal') { } } - const queuesToStart = ['db_queue'] + const queuesToStart: string[] = [] if (workersOption === 'agent' || workersOption === 'ai') { queuesToStart.push('agent_queue') } else if (workersOption === 'transcode') { diff --git a/build.ts b/build.ts index 3ad7abb9..25ce4446 100644 --- a/build.ts +++ b/build.ts @@ -3,7 +3,6 @@ import { bunPluginPino } from 'bun-plugin-pino' import tailwindPlugin from 'bun-plugin-tailwind' import { existsSync } from 'fs' import { rm } from 'fs/promises' -import { resolve } from 'path' import { temporalWorkflow } from './packages/workflow-core/src/bun-temporal-plugin' console.log('\nšŸš€ Starting build process...\n') diff --git a/bun.lock b/bun.lock index cdb858f5..61c46f14 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,9 @@ "workspaces": { "": { "name": "shumai", + "dependencies": { + "jittered-fractional-indexing": "^1.0.1", + }, "devDependencies": { "@eslint/js": "^9.39.1", "@testcontainers/postgresql": "^11.14.0", @@ -61,7 +64,6 @@ "@shumai/dtos": "workspace:*", "@shumai/transcode": "workspace:*", "@shumai/webui": "workspace:*", - "@shumai/worker-db": "workspace:*", "@shumai/workflow-core": "workspace:*", "hono": "^4.12.14", }, @@ -267,22 +269,6 @@ "vite-tsconfig-paths": "^6.1.1", }, }, - "packages/worker-db": { - "name": "@shumai/worker-db", - "version": "0.1.0", - "dependencies": { - "@earendil-works/pi-agent-core": "0.78.0", - "@shumai/core": "workspace:*", - "@shumai/db": "workspace:*", - "@shumai/dtos": "workspace:*", - "@shumai/workflow-core": "workspace:*", - "@temporalio/activity": "^1.16.1", - "@temporalio/worker": "^1.16.1", - "@temporalio/workflow": "^1.16.1", - "jittered-fractional-indexing": "^1.0.1", - "ulid": "^3.0.2", - }, - }, "packages/workflow-core": { "name": "@shumai/workflow-core", "version": "0.1.0", @@ -1017,8 +1003,6 @@ "@shumai/webui": ["@shumai/webui@workspace:packages/webui"], - "@shumai/worker-db": ["@shumai/worker-db@workspace:packages/worker-db"], - "@shumai/workflow-core": ["@shumai/workflow-core@workspace:packages/workflow-core"], "@sinclair/typebox": ["@sinclair/typebox@0.34.49", "", {}, "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A=="], diff --git a/package.json b/package.json index 59fbcccb..e29ecab1 100644 --- a/package.json +++ b/package.json @@ -40,5 +40,8 @@ "typescript-eslint": "^8.46.4", "vitest": "^4.1.4", "vite-tsconfig-paths": "^6.1.1" + }, + "dependencies": { + "jittered-fractional-indexing": "^1.0.1" } } diff --git a/packages/agent/src/activities/agent.ts b/packages/agent/src/activities/agent.ts index 925ff626..6032bbe5 100644 --- a/packages/agent/src/activities/agent.ts +++ b/packages/agent/src/activities/agent.ts @@ -1,11 +1,21 @@ import { createAgentSession, fieldsToTypeBoxSchema, type AutofillField } from '../index' import { DatabaseSessionStorage } from '../database-session-storage' -import type { Prisma, Skill } from '@shumai/db' +import { prisma, AssetType, Prisma, type Skill } from '@shumai/db' import { logger } from '@shumai/core/src/logger' import { s3Service } from '@shumai/core/src/s3/s3' -import { type AgentTool } from '@earendil-works/pi-agent-core' +import { type AgentTool, type AgentMessage, type SessionTreeEntry } from '@earendil-works/pi-agent-core' import { type ImageContent } from '@earendil-works/pi-ai' import { ApplicationFailure } from '@temporalio/activity' +import { assetService } from '@shumai/core/src/asset/asset' +import { VersionStackService } from '@shumai/core/src/versionStack/versionStack' +import { uploadService } from '@shumai/core/src/upload/upload' +import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz' +import { generateKeyBetween } from 'jittered-fractional-indexing' +import { paginateQuery, encodeCursor } from '@shumai/core/src/pagination' +import { ulid } from 'ulid' +import { metadataService } from '@shumai/core/src/metadata/metadata' + +import { UpdateAssetMetadataRequest } from '@shumai/dtos' export interface Usage { inputTokens: number @@ -244,3 +254,846 @@ export async function autofillAiActivity(params: AutofillAiParams) { sessionId, } } + +export async function initializeAgentSessionActivity(params: { + teamId: string + agentId: string + userCommentId: string + userId?: string +}): Promise { + const agentId = params.agentId === 'default' ? 'default' : params.agentId + + // Ensure user exists for agent + const userExists = await prisma.user.findUnique({ where: { id: agentId } }) + if (!userExists) { + await prisma.user.create({ + data: { + id: agentId, + name: 'Ai Agent', + email: `${agentId}@shumai.ai`, + type: 'agent', + }, + }) + } + + // Ensure agent exists + const agentExists = await prisma.agent.findUnique({ where: { id: agentId } }) + if (!agentExists) { + await prisma.agent.create({ + data: { + id: agentId, + teamId: params.teamId, + type: 'chat', + config: { + provider: 'openai', + model: 'gpt-4', + }, + }, + }) + } + + // Create a new AgentSession + const newSession = await prisma.agentSession.create({ + data: { + agentId, + userId: params.userId || null, + cwd: process.cwd(), + }, + }) + const sessionId = newSession.id + + // Fetch existing comments as context + let existingComments: Array<{ + id: string + message: string | null + createdAt: Date + creatorId: string | null + creator: { type: string; name: string } | null + sessionId: string | null + }> = [] + + const userComment = await prisma.assetComment.findUnique({ + where: { id: params.userCommentId }, + include: { creator: true }, + }) + + if (userComment) { + if (!userComment.replyToId) { + // Rule 1: outside of a reply, add all other comments on that asset + existingComments = await prisma.assetComment.findMany({ + where: { + assetId: userComment.assetId, + id: { not: userComment.id }, + }, + orderBy: { id: 'asc' }, + include: { creator: true }, + }) + } else { + // Rule 2: in a reply, add root comment + all other replies to it + const rootComment = await prisma.assetComment.findUnique({ + where: { id: userComment.replyToId }, + include: { creator: true }, + }) + if (rootComment) { + const replies = await prisma.assetComment.findMany({ + where: { + replyToId: rootComment.id, + id: { not: userComment.id }, + }, + orderBy: { id: 'asc' }, + include: { creator: true }, + }) + existingComments = [rootComment, ...replies] + } + } + } + + // Resolve user mentions from IDs to names + const mentionRegex = /<@([^>]+)>/g + const mentionedUserIds = new Set() + for (const c of existingComments) { + if (c.message) { + const matches = [...c.message.matchAll(mentionRegex)] + for (const match of matches) { + mentionedUserIds.add(match[1]) + } + } + } + + const userIdToNameMap = new Map() + if (mentionedUserIds.size > 0) { + const resolvedUsers = await prisma.user.findMany({ + where: { + id: { in: Array.from(mentionedUserIds) }, + }, + select: { + id: true, + name: true, + }, + }) + for (const u of resolvedUsers) { + if (u.name) { + userIdToNameMap.set(u.id, u.name) + } + } + } + + // Save context as AgentSessionEntry records + let prevId: string | null = null + for (const c of existingComments) { + const isAgent = c.creator?.type === 'agent' || c.sessionId !== null + let messageContent = c.message || '' + + // Replace <@USER_ID> with <@USER_NAME> + messageContent = messageContent.replace(mentionRegex, (match, userId) => { + const resolvedName = userIdToNameMap.get(userId) + return resolvedName ? `<@${resolvedName}>` : match + }) + + if (isAgent) { + const agentName = c.creator?.name || 'Ai Agent' + messageContent = `[Agent Message][${agentName}]: ${messageContent}` + } else if (c.creator?.name) { + messageContent = `[${c.creator.name}]: ${messageContent}` + } + + const entryId = ulid() + const message: AgentMessage = { + role: 'user', + content: [{ type: 'text', text: messageContent }], + timestamp: c.createdAt.getTime(), + } + + const entryJson: SessionTreeEntry = { + type: 'message', + id: entryId, + parentId: prevId, + timestamp: c.createdAt.toISOString(), + message, + } + + await prisma.agentSessionEntry.create({ + data: { + id: entryId, + sessionId, + entry: entryJson as unknown as PrismaJson.PiSessionEntry, + }, + }) + prevId = entryId + } + + // Update leafId of the session + if (prevId) { + await prisma.agentSession.update({ + where: { id: sessionId }, + data: { leafId: prevId }, + }) + } + + return sessionId +} + +export async function deleteCommentActivity(commentId: string) { + return prisma.assetComment.delete({ + where: { id: commentId }, + }) +} + +export async function createCommentActivity(params: { + assetId: string + message: string + sessionId?: string | null + agentId?: string | null + replyToId?: string | null +}) { + if (params.sessionId) { + const agentId = params.agentId && params.agentId !== 'default' ? params.agentId : 'default' + + // Fetch teamId from asset + const asset = await prisma.asset.findUnique({ + where: { id: params.assetId }, + include: { project: true }, + }) + if (!asset || !asset.project) { + throw ApplicationFailure.create({ message: 'asset or project not found', nonRetryable: true }) + } + const teamId = asset.project.teamId + + // Ensure user exists for agent + const userExists = await prisma.user.findUnique({ where: { id: agentId } }) + if (!userExists) { + await prisma.user.create({ + data: { + id: agentId, + name: 'Ai Agent', + email: `${agentId}@shumai.ai`, + type: 'agent', + }, + }) + } + + // Ensure agent exists + const agentExists = await prisma.agent.findUnique({ where: { id: agentId } }) + if (!agentExists) { + await prisma.agent.create({ + data: { + id: agentId, + teamId, + type: 'chat', + config: { + provider: 'openai', + model: 'gpt-4', + }, + }, + }) + } + + // Ensure agent session exists + const sessionExists = await prisma.agentSession.findUnique({ where: { id: params.sessionId } }) + if (!sessionExists) { + await prisma.agentSession.create({ + data: { + id: params.sessionId, + agentId: agentId, + cwd: process.cwd(), + }, + }) + } + } + + return prisma.assetComment.create({ + data: { + assetId: params.assetId, + message: params.message, + sessionId: params.sessionId ?? null, + creatorId: params.agentId && params.agentId !== 'default' ? params.agentId : null, + replyToId: params.replyToId, + }, + }) +} + +export async function updateCommentActivity(params: { + commentId: string + message: string + sessionId?: string | null +}) { + return prisma.assetComment.update({ + where: { id: params.commentId }, + data: { + message: params.message, + ...(params.sessionId !== undefined ? { sessionId: params.sessionId } : {}), + }, + }) +} + +export async function getAgentChatContextActivity(params: { + teamId: string + agentId: string +}): Promise { + const team = await prisma.team.findUnique({ + where: { id: params.teamId }, + }) + if (!team) { + throw ApplicationFailure.create({ message: 'failed to get team', nonRetryable: true }) + } + + const agent = await prisma.agent.findUnique({ + where: { id: params.agentId }, + include: { + provider: true, + modelRef: true, + }, + }) + if (!agent) { + throw ApplicationFailure.create({ + message: `agent ${params.agentId} not found`, + nonRetryable: true, + }) + } + + if (!agent.provider) { + throw ApplicationFailure.create({ + message: 'agent has no provider configured', + nonRetryable: true, + }) + } + if (!agent.modelRef) { + throw ApplicationFailure.create({ + message: 'agent has no model configured', + nonRetryable: true, + }) + } + + const providerName = agent.provider.name + + // Fetch the required provider configuration from database + const dbProviders = await prisma.provider.findMany({ + where: { teamId: params.teamId, name: providerName }, + include: { models: true }, + }) + + // Fetch team skills + const teamSkills = await prisma.skill.findMany({ + where: { teamId: params.teamId }, + }) + + const sandbox = await prisma.sandbox.findUnique({ + where: { teamId: params.teamId }, + }) + const allowedDomains = sandbox?.allowedDomains || [] + + return { + agent, + dbProviders, + teamSkills, + allowedDomains, + } +} + +export async function getAgentAutofillContextActivity(params: { + teamId: string +}): Promise { + const agent = await prisma.agent.findFirst({ + where: { + type: 'autofill', + enabled: true, + teamId: params.teamId, + }, + }) + if (!agent) { + throw ApplicationFailure.create({ + message: 'no autofill agent found for team', + nonRetryable: true, + }) + } + + const team = await prisma.team.findUnique({ + where: { id: params.teamId }, + }) + if (!team) { + throw ApplicationFailure.create({ message: 'failed to get team', nonRetryable: true }) + } + + // Find provider/model configuration + const agentWithDetails = await prisma.agent.findUnique({ + where: { id: agent.id }, + include: { + provider: true, + modelRef: true, + }, + }) + if (!agentWithDetails) { + throw ApplicationFailure.create({ message: `agent ${agent.id} not found`, nonRetryable: true }) + } + if (!agentWithDetails.provider) { + throw ApplicationFailure.create({ + message: 'agent has no provider configured', + nonRetryable: true, + }) + } + if (!agentWithDetails.modelRef) { + throw ApplicationFailure.create({ + message: 'agent has no model configured', + nonRetryable: true, + }) + } + + const providerName = agentWithDetails.provider.name + + // Fetch the required provider configuration from database + const dbProviders = await prisma.provider.findMany({ + where: { teamId: params.teamId, name: providerName }, + include: { models: true }, + }) + + // Fetch team skills + const teamSkills = await prisma.skill.findMany({ + where: { teamId: params.teamId }, + }) + + const sandbox = await prisma.sandbox.findUnique({ + where: { teamId: params.teamId }, + }) + const allowedDomains = sandbox?.allowedDomains || [] + + return { + agent: agentWithDetails, + dbProviders, + teamSkills, + allowedDomains, + } +} + +export async function getAssetActivity(assetId: string) { + return prisma.asset.findUnique({ + where: { id: assetId }, + include: { + storageKey: true, + project: { + include: { + team: true, + }, + }, + }, + }) +} + +export async function getCommentActivity(commentId: string) { + return prisma.assetComment.findUnique({ + where: { id: commentId }, + include: { attachments: { include: { asset: { include: { storageKey: true } } } } }, + }) +} + +export async function getProjectAutofillFieldsActivity(projectId: string) { + const fields = await metadataService.listProjectFields('', projectId) + return fields.filter((f) => f.field.aiAutofill).map((f) => f.field) +} + +export async function updateAssetMetadataActivity(params: { + assetId: string + metadata: UpdateAssetMetadataRequest[] +}) { + return metadataService.updateAssetMetadata(params.assetId, params.metadata) +} + +export async function getAssetPathContextActivity(assetId: string): Promise { + const parts: { name: string; id: string }[] = [] + let currentId: string | null = assetId + + while (currentId) { + /* prettier-ignore */ + const assetNode = (await prisma.asset.findUnique({ + where: { id: currentId }, + select: { id: true, name: true, parentId: true, type: true }, + })) as { id: string; name: string; parentId: string | null; type: AssetType } | null + if (!assetNode) break + + if (assetNode.name) { + parts.unshift({ name: assetNode.name, id: assetNode.id }) + } + + currentId = assetNode.parentId + } + + if (parts.length === 0) return '' + + const pathStr = parts.map((p) => p.name).join('/') + let contextStr = `Path: ${pathStr}\n\n` + for (const part of parts) { + contextStr += `name: ${part.name}, id: ${part.id}\n` + } + + return contextStr +} + +export interface ExecuteAgentToolParams { + taskId: string + toolName: string + // args are dynamic JSON arguments passed from the agent harness at runtime + // eslint-disable-next-line @typescript-eslint/no-explicit-any + args: any + userId?: string +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function executeAgentToolActivity(params: ExecuteAgentToolParams): Promise { + const { toolName, args, userId } = params + if (!userId) { + throw ApplicationFailure.create({ + message: 'User ID is required for tool authz', + nonRetryable: true, + }) + } + + // 1. Fetch User + const user = await prisma.user.findUnique({ where: { id: userId } }) + if (!user) { + throw ApplicationFailure.create({ + message: `User not found: ${userId}`, + nonRetryable: true, + }) + } + + // 2. Perform Authz and Action based on toolName + switch (toolName) { + case 'list_assets': { + const parent = args.parent + if (!parent) { + throw ApplicationFailure.create({ + message: 'parent parameter is required', + nonRetryable: true, + }) + } + + // Authz check + await authzService.hasPermission({ + user, + permission: Permission.Read, + type: ResourceType.Asset, + id: parent, + }) + + const page = args.page || 1 + const pageSize = args.pageSize || 20 + const type = args.type || 'all' + + // We need to list files and folders of parent dir + const where: Prisma.AssetWhereInput = { + parentId: parent, + isDeleted: false, + } + if (type === 'file') { + where.type = { in: ['file', 'version_stack'] } + } else if (type === 'folder') { + where.type = 'folder' + } else { + where.type = { in: ['file', 'folder', 'version_stack'] } + } + + const limit = pageSize + const offset = (page - 1) * limit + const after = offset > 0 ? encodeCursor(offset) : undefined + + const { data: assets, pageInfo } = await paginateQuery< + Prisma.AssetGetPayload> + >( + async (skip, take) => { + return prisma.asset.findMany({ + where, + orderBy: { id: 'desc' }, // sort desc per ULID default in sorting guidelines! + skip, + take, + }) + }, + async () => prisma.asset.count({ where }), + { first: limit, after, includeCount: true }, + ) + + // return asset id, asset name, asset type, asset size (sizeByte or size) + const results = assets.map((a) => ({ + id: a.id, + name: a.name, + type: a.type, + size: a.sizeByte, + })) + + return { + assets: results, + pageInfo, + } + } + + case 'create_folder': { + const parent = args.parent + const name = args.name + if (!parent || !name) { + throw ApplicationFailure.create({ + message: 'parent and name parameters are required', + nonRetryable: true, + }) + } + + // Authz check + await authzService.hasPermission({ + user, + permission: Permission.Edit, + type: ResourceType.Asset, + id: parent, + }) + + const newFolder = await assetService.createAsset({ + name, + parentId: parent, + type: 'folder', + creatorId: userId, + }) + + // Increment fileCount of parent folder + await prisma.asset.update({ + where: { id: parent }, + data: { fileCount: { increment: 1 } }, + }) + + return { + id: newFolder.id, + name: newFolder.name, + type: newFolder.type, + size: newFolder.sizeByte, + } + } + + case 'create_file': { + const parent = args.parent + const s3Key = args.s3Key as string + const name = args.name as string + const fileSize = args.size as number + const mimeType = args.contentType as string + if (!parent || !s3Key || !name || fileSize === undefined || !mimeType) { + throw ApplicationFailure.create({ + message: 'parent, s3Key, name, size, and contentType parameters are required', + nonRetryable: true, + }) + } + + // Authz check + await authzService.hasPermission({ + user, + permission: Permission.Edit, + type: ResourceType.Asset, + id: parent, + }) + + // Fetch parent folder to resolve projectId and teamId + const parentAsset = await prisma.asset.findUnique({ + where: { id: parent }, + include: { project: true }, + }) + if (!parentAsset) { + throw ApplicationFailure.create({ + message: `Parent folder not found with ID: ${parent}`, + nonRetryable: true, + }) + } + if (!parentAsset.projectId || !parentAsset.project?.teamId) { + throw ApplicationFailure.create({ + message: `Parent folder ${parent} has no project or team associated`, + nonRetryable: true, + }) + } + + // Create asset via assetService + const newFile = await assetService.createAsset({ + name, + type: 'file', + parentId: parent, + key: s3Key, + sizeByte: fileSize, + contentType: mimeType, + creatorId: userId, + }) + + // Increment fileCount of parent and update ancestor size + await prisma.$transaction(async (tx) => { + await tx.asset.update({ + where: { id: parent }, + data: { fileCount: { increment: 1 } }, + }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await assetService.updateAncestorsSize(tx as any, parent, fileSize) + + // Trigger post-upload transcode and AI workflows + await uploadService.triggerPostUploadWorkflows( + tx, + newFile.id, + parentAsset.project!.teamId, + parentAsset.projectId!, + ) + }) + + return { + id: newFile.id, + name: newFile.name, + type: newFile.type, + size: newFile.sizeByte, + } + } + + case 'create_version': { + const parent = args.parent // parent file id + const s3Key = args.s3Key as string + const name = args.name as string + const fileSize = args.size as number + const mimeType = args.contentType as string + if (!parent || !s3Key || !name || fileSize === undefined || !mimeType) { + throw ApplicationFailure.create({ + message: 'parent, s3Key, name, size, and contentType parameters are required', + nonRetryable: true, + }) + } + + // Fetch parent file and cast to any due to Prisma type resolution limits + /* prettier-ignore */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const parentFile = (await prisma.asset.findUnique({ where: { id: parent }, include: { parent: true, project: true } })) as any + if (!parentFile) { + throw ApplicationFailure.create({ + message: `Parent file not found with ID: ${parent}`, + nonRetryable: true, + }) + } + + if (!parentFile.projectId || !parentFile.project?.teamId) { + throw ApplicationFailure.create({ + message: `Parent file ${parent} has no project or team associated`, + nonRetryable: true, + }) + } + + // Authz check: verify Edit permission on the parent file (resource ID is parent file id) + await authzService.hasPermission({ + user, + permission: Permission.Edit, + type: ResourceType.Asset, + id: parent, + }) + + // If already in a version stack + if (parentFile.parentId && parentFile.parent?.type === AssetType.version_stack) { + const stackId = parentFile.parentId + + // Create the new file asset + const newFile = await assetService.createAsset({ + name, + type: 'file', + parentId: stackId, + key: s3Key, + sizeByte: fileSize, + contentType: mimeType, + creatorId: userId, + }) + + // Generate sort index for new version inside the stack (newest gets the lowest sortIndex) + const firstChild = await prisma.asset.findFirst({ + where: { + parentId: stackId, + NOT: { id: newFile.id }, + }, + orderBy: { sortIndex: 'asc' }, + }) + const newSortIndex = generateKeyBetween(null, firstChild?.sortIndex || null) + + await prisma.$transaction(async (tx) => { + // Assign sort index + await tx.asset.update({ + where: { id: newFile.id }, + data: { sortIndex: newSortIndex }, + }) + // Increment stack's fileCount and size + const updatedStack = await tx.asset.update({ + where: { id: stackId }, + data: { + fileCount: { increment: 1 }, + sizeByte: { increment: fileSize }, + }, + }) + // Update ancestors' size of stack's parent + if (updatedStack.parentId) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await assetService.updateAncestorsSize(tx as any, updatedStack.parentId, fileSize) + } + + // Trigger post-upload transcode and AI workflows + await uploadService.triggerPostUploadWorkflows( + tx, + newFile.id, + parentFile.project.teamId, + parentFile.projectId, + ) + }) + + return { + id: newFile.id, + name: newFile.name, + type: newFile.type, + size: newFile.sizeByte, + } + } else { + // Parent is a regular file. We must create a new version stack. + const folderParentId = parentFile.parentId + if (!folderParentId) { + throw ApplicationFailure.create({ + message: `Parent file ${parent} has no parent folder`, + nonRetryable: true, + }) + } + + // Create the new file asset under parent folder first (so that createVersionStack can run on them, + // which enforces both files must have same parent initially) + const newFile = await assetService.createAsset({ + name, + type: 'file', + parentId: folderParentId, + key: s3Key, + sizeByte: fileSize, + contentType: mimeType, + creatorId: userId, + }) + + await prisma.$transaction(async (tx) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const txVersionStackService = new VersionStackService(tx as any) + await txVersionStackService.createVersionStack({ + fileIds: [newFile.id, parentFile.id], + projectId: parentFile.projectId!, + creatorId: userId, + }) + // Update ancestors' size (add size of the new file version) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await assetService.updateAncestorsSize(tx as any, folderParentId!, fileSize) + + // Trigger post-upload transcode and AI workflows + await uploadService.triggerPostUploadWorkflows( + tx, + newFile.id, + parentFile.project.teamId, + parentFile.projectId!, + ) + }) + + return { + id: newFile.id, + name: newFile.name, + type: newFile.type, + size: newFile.sizeByte, + } + } + } + + default: + throw ApplicationFailure.create({ + message: `Unsupported tool name: ${toolName}`, + nonRetryable: true, + }) + } +} diff --git a/packages/agent/src/activities/ai.ts b/packages/agent/src/activities/ai.ts index 19f422a3..26a18567 100644 --- a/packages/agent/src/activities/ai.ts +++ b/packages/agent/src/activities/ai.ts @@ -7,6 +7,7 @@ import * as os from 'os' import * as path from 'path' import { promisify } from 'util' import { ApplicationFailure } from '@temporalio/activity' +import { prisma } from '@shumai/db' const execAsync = promisify(exec) @@ -259,3 +260,69 @@ export async function extractAiMetadataActivity( return generatedFiles } + +export async function getEmbeddingContextActivity(params: { + teamId: string + assetId: string +}) { + const team = await prisma.team.findUnique({ + where: { id: params.teamId }, + }) + if (!team) { + throw ApplicationFailure.create({ message: 'failed to get team', nonRetryable: true }) + } + + const agent = await prisma.agent.findFirst({ + where: { + type: 'embedding', + enabled: true, + user: { teamMembers: { some: { teamId: params.teamId } } }, + }, + }) + if (!agent) { + throw ApplicationFailure.create({ + message: 'embedding feature is disabled or agent not found', + nonRetryable: true, + }) + } + + const asset = await prisma.asset.findUnique({ + where: { id: params.assetId }, + include: { storageKey: true }, + }) + if (!asset) { + throw ApplicationFailure.create({ message: 'failed to get asset', nonRetryable: true }) + } + if (!asset.mediaType) { + throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true }) + } + + return { + agent, + asset, + } +} + +export async function saveAssetEmbeddingsActivity(params: { + assetId: string + embeddings: Array<{ + embedding: number[] + startTime?: number + endTime?: number + }> +}) { + for (const item of params.embeddings) { + const embVec = JSON.stringify(item.embedding) + if (item.startTime !== undefined && item.endTime !== undefined) { + await prisma.$executeRaw` + INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at) + VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW()) + ` + } else { + await prisma.$executeRaw` + INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at) + VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW()) + ` + } + } +} diff --git a/packages/agent/src/workflows/agent-autofill.ts b/packages/agent/src/workflows/agent-autofill.ts index f6de1e23..f0c9e173 100644 --- a/packages/agent/src/workflows/agent-autofill.ts +++ b/packages/agent/src/workflows/agent-autofill.ts @@ -3,7 +3,6 @@ import type { WorkflowTask } from '@shumai/db' import { getActivities, executeActivity, - TaskQueueDb, TaskQueueAgent, TaskQueueTranscode, } from '@shumai/workflow-core' @@ -32,14 +31,14 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { try { // Update status to processing - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) // 0. Create Placeholder Comment const payload = task.payload - const placeholder = await executeActivity(TaskQueueDb, createCommentActivity, { + const placeholder = await executeActivity(TaskQueueAgent, createCommentActivity, { assetId: task.assetId, message: '__AUTOFILL__', sessionId: payload?.agent?.sessionId || task.id, @@ -48,7 +47,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { placeholderCommentId = placeholder.id // 1. Get Asset - const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId) + const asset = await executeActivity(TaskQueueAgent, getAssetActivity, task.assetId) const key = asset?.storageKey?.key if (!asset || !asset.project || !key) { throw ApplicationFailure.create({ message: 'Asset or project not found', nonRetryable: true }) @@ -81,12 +80,12 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { if (generatedFiles.length === 0) { // If no images could be extracted, we can't do much if (placeholderCommentId) { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: 'Autofill completed: No images could be extracted for analysis.', }) } - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -94,23 +93,23 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { } // 3. Get Project Autofill Fields - const fields = await executeActivity(TaskQueueDb, getProjectAutofillFieldsActivity, projectId) + const fields = await executeActivity(TaskQueueAgent, getProjectAutofillFieldsActivity, projectId) if (fields.length === 0) { if (placeholderCommentId) { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: 'Autofill completed: No autofill fields defined in project.', }) } - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) return } - // 3b. Fetch Agent Context (Database Activity on db_queue) - const context = await executeActivity(TaskQueueDb, getAgentAutofillContextActivity, { + // 3b. Fetch Agent Context + const context = await executeActivity(TaskQueueAgent, getAgentAutofillContextActivity, { teamId, }) @@ -132,7 +131,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { if (aiResult.usage) { // Update Usage - await executeActivity(TaskQueueDb, updateTaskUsageActivity, { + await executeActivity(TaskQueueAgent, updateTaskUsageActivity, { taskId: task.id, inputTokens: aiResult.usage.inputTokens, outputTokens: aiResult.usage.outputTokens, @@ -148,7 +147,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { })) if (metadataUpdates.length > 0) { - await executeActivity(TaskQueueDb, updateAssetMetadataActivity, { + await executeActivity(TaskQueueAgent, updateAssetMetadataActivity, { assetId: asset.id, metadata: metadataUpdates, }) @@ -156,7 +155,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { // 7. Update Placeholder Comment if (placeholderCommentId) { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: 'Autofill completed successfully.', sessionId: aiResult.sessionId, @@ -164,7 +163,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { } // Update status to completed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -174,7 +173,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { // Update placeholder comment with error message if (placeholderCommentId) { try { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: `Autofill failed: ${err instanceof Error ? err.message : String(err)}`, }) @@ -184,7 +183,7 @@ export async function agentAutofillMedia(task: WorkflowTask): Promise { } // Update status to failed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'failed', output: { error: err instanceof Error ? err.message : String(err) }, diff --git a/packages/agent/src/workflows/agent-chat.ts b/packages/agent/src/workflows/agent-chat.ts index 7228d1e5..216bd82c 100644 --- a/packages/agent/src/workflows/agent-chat.ts +++ b/packages/agent/src/workflows/agent-chat.ts @@ -1,6 +1,6 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { executeActivity, getActivities, TaskQueueAgent, TaskQueueDb } from '@shumai/workflow-core' +import { executeActivity, getActivities, TaskQueueAgent } from '@shumai/workflow-core' export async function agentChat(task: WorkflowTask): Promise { const { @@ -22,7 +22,7 @@ export async function agentChat(task: WorkflowTask): Promise { try { // Update status to processing - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) @@ -47,13 +47,13 @@ export async function agentChat(task: WorkflowTask): Promise { nonRetryable: true, }) } - const userComment = await executeActivity(TaskQueueDb, getCommentActivity, userCommentId) + const userComment = await executeActivity(TaskQueueAgent, getCommentActivity, userCommentId) if (!userComment) { throw ApplicationFailure.create({ message: 'User comment not found', nonRetryable: true }) } // 2. Create Placeholder Comment - const placeholder = await executeActivity(TaskQueueDb, createCommentActivity, { + const placeholder = await executeActivity(TaskQueueAgent, createCommentActivity, { assetId: task.assetId, message: '__CHAT__', sessionId: sessionId || 'pending', @@ -63,15 +63,15 @@ export async function agentChat(task: WorkflowTask): Promise { placeholderCommentId = placeholder.id // 3. Get Asset - const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId) + const asset = await executeActivity(TaskQueueAgent, getAssetActivity, task.assetId) if (!asset || !asset.project) { throw ApplicationFailure.create({ message: 'Asset or project not found', nonRetryable: true }) } const teamId = asset.project.teamId - // 4. Initialize Session if missing (Database Activity on db_queue) + // 4. Initialize Session if missing if (!sessionId) { - sessionId = await executeActivity(TaskQueueDb, initializeAgentSessionActivity, { + sessionId = await executeActivity(TaskQueueAgent, initializeAgentSessionActivity, { teamId, agentId: agentId, userCommentId, @@ -79,8 +79,8 @@ export async function agentChat(task: WorkflowTask): Promise { }) } - // 4b. Fetch Agent Context (Database Activity on db_queue) - const context = await executeActivity(TaskQueueDb, getAgentChatContextActivity, { + // 4b. Fetch Agent Context + const context = await executeActivity(TaskQueueAgent, getAgentChatContextActivity, { teamId, agentId: agentId, }) @@ -96,7 +96,7 @@ export async function agentChat(task: WorkflowTask): Promise { } const pathContext = await executeActivity( - TaskQueueDb, + TaskQueueAgent, getAssetPathContextActivity, task.assetId, ) @@ -145,9 +145,9 @@ export async function agentChat(task: WorkflowTask): Promise { // 7. Update Placeholder Comment if (placeholderCommentId) { if (aiResult.text.trim() === '__NO_REPLY__') { - await executeActivity(TaskQueueDb, deleteCommentActivity, placeholderCommentId) + await executeActivity(TaskQueueAgent, deleteCommentActivity, placeholderCommentId) } else { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: aiResult.text, sessionId: aiResult.sessionId, @@ -157,7 +157,7 @@ export async function agentChat(task: WorkflowTask): Promise { // 8. Update Usage if (aiResult.usage) { - await executeActivity(TaskQueueDb, updateTaskUsageActivity, { + await executeActivity(TaskQueueAgent, updateTaskUsageActivity, { taskId: task.id, inputTokens: aiResult.usage.inputTokens, outputTokens: aiResult.usage.outputTokens, @@ -166,7 +166,7 @@ export async function agentChat(task: WorkflowTask): Promise { } // Update status to completed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'completed', output: { sessionId: aiResult.sessionId }, @@ -182,7 +182,7 @@ export async function agentChat(task: WorkflowTask): Promise { ? `AI error: ${err.message.substring(9)}` : 'Sorry, I encountered an error while processing your request.' - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: errorMessage, }) @@ -192,7 +192,7 @@ export async function agentChat(task: WorkflowTask): Promise { } // Update status to failed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'failed', output: { error: err instanceof Error ? err.message : String(err) }, diff --git a/packages/agent/src/workflows/agent-embedding.ts b/packages/agent/src/workflows/agent-embedding.ts index 4cd38e46..19acc5af 100644 --- a/packages/agent/src/workflows/agent-embedding.ts +++ b/packages/agent/src/workflows/agent-embedding.ts @@ -1,6 +1,6 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueDb, TaskQueueAgent } from '@shumai/workflow-core' +import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core' export async function agentEmbeddingMedia(task: WorkflowTask): Promise { const { @@ -18,14 +18,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { try { // Update status to processing - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) // 0. Create Placeholder Comment const payload = task.payload - const placeholder = await executeActivity(TaskQueueDb, createCommentActivity, { + const placeholder = await executeActivity(TaskQueueAgent, createCommentActivity, { assetId: task.assetId, message: '__EMBEDDING__', sessionId: payload?.agent?.sessionId || task.id, @@ -37,8 +37,8 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { throw ApplicationFailure.create({ message: 'Task has no teamId', nonRetryable: true }) } - // 1. Fetch Agent Context (Database Activity on db_queue) - const context = await executeActivity(TaskQueueDb, getEmbeddingContextActivity, { + // 1. Fetch Agent Context + const context = await executeActivity(TaskQueueAgent, getEmbeddingContextActivity, { teamId: task.teamId, assetId: task.assetId, }) @@ -54,7 +54,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // Save computed embeddings if (result.embeddings.length > 0) { - await executeActivity(TaskQueueDb, saveAssetEmbeddingsActivity, { + await executeActivity(TaskQueueAgent, saveAssetEmbeddingsActivity, { assetId: task.assetId, embeddings: result.embeddings, }) @@ -62,7 +62,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // Update Usage if (result.usage) { - await executeActivity(TaskQueueDb, updateTaskUsageActivity, { + await executeActivity(TaskQueueAgent, updateTaskUsageActivity, { taskId: task.id, inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens, @@ -72,14 +72,14 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // 7. Update Placeholder Comment if (placeholderCommentId) { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: 'Embedding completed successfully.', }) } // Update status to completed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) @@ -89,7 +89,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { // Update placeholder comment with error message if (placeholderCommentId) { try { - await executeActivity(TaskQueueDb, updateCommentActivity, { + await executeActivity(TaskQueueAgent, updateCommentActivity, { commentId: placeholderCommentId, message: `Embedding failed: ${err instanceof Error ? err.message : String(err)}`, }) @@ -99,7 +99,7 @@ export async function agentEmbeddingMedia(task: WorkflowTask): Promise { } // Update status to failed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'failed', output: { error: err instanceof Error ? err.message : String(err) }, diff --git a/packages/agent/src/workflows/agent-tool-call.ts b/packages/agent/src/workflows/agent-tool-call.ts index b7b7817f..0a69bddc 100644 --- a/packages/agent/src/workflows/agent-tool-call.ts +++ b/packages/agent/src/workflows/agent-tool-call.ts @@ -1,13 +1,13 @@ import { ApplicationFailure } from '@temporalio/workflow' import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueDb } from '@shumai/workflow-core' +import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core' export async function agentToolCall(task: WorkflowTask): Promise { const { updateTaskStatusActivity, executeAgentToolActivity } = getActivities() try { // Update status to processing - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) @@ -22,8 +22,8 @@ export async function agentToolCall(task: WorkflowTask): Promise { const { toolName, args, userId } = payload.agentToolCall - // Execute the database activity on db_queue - const result = await executeActivity(TaskQueueDb, executeAgentToolActivity, { + // Execute the agent activity on agent_queue + const result = await executeActivity(TaskQueueAgent, executeAgentToolActivity, { taskId: task.id, toolName, args, @@ -31,7 +31,7 @@ export async function agentToolCall(task: WorkflowTask): Promise { }) // Update status to completed with output - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'completed', output: result, @@ -40,7 +40,7 @@ export async function agentToolCall(task: WorkflowTask): Promise { console.error(`AgentToolCall failed for task ${task.id}:`, err) // Update status to failed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueAgent, updateTaskStatusActivity, { taskId: task.id, status: 'failed', output: { error: err instanceof Error ? err.message : String(err) }, diff --git a/packages/agent/src/workflows/query-embedding-for-search.ts b/packages/agent/src/workflows/query-embedding-for-search.ts index 2acc1a6c..edd8fd1c 100644 --- a/packages/agent/src/workflows/query-embedding-for-search.ts +++ b/packages/agent/src/workflows/query-embedding-for-search.ts @@ -1,5 +1,5 @@ import type { WorkflowTask } from '@shumai/db' -import { getActivities, executeActivity, TaskQueueDb } from '@shumai/workflow-core' +import { getActivities, executeActivity, TaskQueueAgent } from '@shumai/workflow-core' export async function queryEmbeddingForSearch(task: WorkflowTask): Promise { const { generateTextEmbeddingActivity, updateWorkflowTaskActivity } = getActivities() @@ -13,20 +13,20 @@ export async function queryEmbeddingForSearch(task: WorkflowTask): Promise try { // 1. Mark as processing - await executeActivity(TaskQueueDb, updateWorkflowTaskActivity, { + await executeActivity(TaskQueueAgent, updateWorkflowTaskActivity, { taskId: task.id, status: 'processing', heartbeat: true, }) // 2. Generate embedding - const result = await generateTextEmbeddingActivity({ + const result = await executeActivity(TaskQueueAgent, generateTextEmbeddingActivity, { text, teamId: task.teamId!, }) // 3. Save output and complete - await executeActivity(TaskQueueDb, updateWorkflowTaskActivity, { + await executeActivity(TaskQueueAgent, updateWorkflowTaskActivity, { taskId: task.id, status: 'completed', output: { embedding: result.embedding }, @@ -36,7 +36,7 @@ export async function queryEmbeddingForSearch(task: WorkflowTask): Promise }) } catch (err) { console.error(`queryEmbeddingForSearch failed for task ${task.id}:`, err) - await executeActivity(TaskQueueDb, updateWorkflowTaskActivity, { + await executeActivity(TaskQueueAgent, updateWorkflowTaskActivity, { taskId: task.id, status: 'failed', output: { diff --git a/packages/transcode/src/workflows/transcode.ts b/packages/transcode/src/workflows/transcode.ts index 4f8d84dc..41c8d972 100644 --- a/packages/transcode/src/workflows/transcode.ts +++ b/packages/transcode/src/workflows/transcode.ts @@ -3,7 +3,6 @@ import { ApplicationFailure } from '@temporalio/workflow' import { getActivities, executeActivity, - TaskQueueDb, TaskQueueTranscode, } from '@shumai/workflow-core' @@ -27,18 +26,18 @@ export async function transcodeMedia(task: WorkflowTask): Promise { try { // 1. Update status to processing - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueTranscode, updateTaskStatusActivity, { taskId: task.id, status: 'processing', }) - await executeActivity(TaskQueueDb, updateAssetStatusActivity, { + await executeActivity(TaskQueueTranscode, updateAssetStatusActivity, { assetId: task.assetId, status: 'processing', }) // 2. Get Asset - const asset = await executeActivity(TaskQueueDb, getAssetActivity, task.assetId) + const asset = await executeActivity(TaskQueueTranscode, getAssetActivity, task.assetId) const key = asset?.storageKey?.key if (!asset || !key) { throw ApplicationFailure.create({ @@ -174,25 +173,25 @@ export async function transcodeMedia(task: WorkflowTask): Promise { } // 11. Update Asset Media and Status - await executeActivity(TaskQueueDb, updateAssetMediaActivity, { + await executeActivity(TaskQueueTranscode, updateAssetMediaActivity, { assetId: asset.id, mediaInfo, }) - await executeActivity(TaskQueueDb, updateAssetStatusActivity, { + await executeActivity(TaskQueueTranscode, updateAssetStatusActivity, { assetId: asset.id, status: 'processed', }) // 12. Update Task Status - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueTranscode, updateTaskStatusActivity, { taskId: task.id, status: 'completed', }) } catch (err) { console.error(`TranscodeMedia failed for task ${task.id}:`, err) // Update status to failed - await executeActivity(TaskQueueDb, updateTaskStatusActivity, { + await executeActivity(TaskQueueTranscode, updateTaskStatusActivity, { taskId: task.id, status: 'failed', output: { error: err instanceof Error ? err.message : String(err) }, diff --git a/packages/worker-db/package.json b/packages/worker-db/package.json deleted file mode 100644 index 0035ec48..00000000 --- a/packages/worker-db/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@shumai/worker-db", - "version": "0.1.0", - "private": true, - "type": "module", - "main": "./src/index.ts", - "types": "./src/index.ts", - "dependencies": { - "@shumai/db": "workspace:*", - "@shumai/dtos": "workspace:*", - "@shumai/core": "workspace:*", - "@shumai/workflow-core": "workspace:*", - "@temporalio/activity": "^1.16.1", - "@temporalio/worker": "^1.16.1", - "@temporalio/workflow": "^1.16.1", - "ulid": "^3.0.2", - "jittered-fractional-indexing": "^1.0.1", - "@earendil-works/pi-agent-core": "0.78.0" - } -} diff --git a/packages/worker-db/src/activities/db.test.ts b/packages/worker-db/src/activities/db.test.ts deleted file mode 100644 index 0b13068d..00000000 --- a/packages/worker-db/src/activities/db.test.ts +++ /dev/null @@ -1,645 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { setupTestDbHooks } from '@shumai/db' -import { prisma } from '@shumai/db' -import { - createCommentActivity, - updateCommentActivity, - deleteCommentActivity, - initializeAgentSessionActivity, - getAgentChatContextActivity, - getAgentAutofillContextActivity, - getEmbeddingContextActivity, - saveAssetEmbeddingsActivity, - getAssetPathContextActivity, - executeAgentToolActivity, -} from './db' -import type { SessionTreeEntry } from '@earendil-works/pi-agent-core' -import { s3Service } from '@shumai/core/src/s3/s3' - -describe('Database Activities', () => { - setupTestDbHooks() - - it('should support comment CRUD activities (create, update, delete)', async () => { - const team = await prisma.team.create({ data: { name: 't1' } }) - const user = await prisma.user.create({ - data: { name: 'User One', email: 'user1@example.com' }, - }) - const project = await prisma.project.create({ - data: { name: 'p1', teamId: team.id }, - }) - const file = await prisma.asset.create({ - data: { - name: 'test-file', - type: 'file', - projectId: project.id, - creatorId: user.id, - status: 'uploaded', - }, - }) - - // 1. Create Comment - const comment = await createCommentActivity({ - assetId: file.id, - message: 'Initial comment', - }) - expect(comment.id).toBeDefined() - expect(comment.message).toBe('Initial comment') - - // 2. Update Comment - const updated = await updateCommentActivity({ - commentId: comment.id, - message: 'Updated comment', - }) - expect(updated.message).toBe('Updated comment') - - // 3. Delete Comment - await deleteCommentActivity(comment.id) - const found = await prisma.assetComment.findUnique({ - where: { id: comment.id }, - }) - expect(found).toBeNull() - }) - - it('should support multiple comments in the same session without violating unique session_id constraint', async () => { - const team = await prisma.team.create({ data: { name: 't2' } }) - const user = await prisma.user.create({ - data: { name: 'User Two', email: 'user2@example.com' }, - }) - const project = await prisma.project.create({ - data: { name: 'p2', teamId: team.id }, - }) - const file = await prisma.asset.create({ - data: { - name: 'test-file-2', - type: 'file', - projectId: project.id, - creatorId: user.id, - status: 'uploaded', - }, - }) - - const sessionId = 'test-multi-comment-session' - - // 1. Create first agent comment in the session - const c1 = await createCommentActivity({ - assetId: file.id, - message: 'Agent comment 1', - sessionId, - agentId: 'default', - }) - expect(c1.id).toBeDefined() - expect(c1.sessionId).toBe(sessionId) - - // 2. Create second agent comment in the SAME session - const c2 = await createCommentActivity({ - assetId: file.id, - message: 'Agent comment 2', - sessionId, - agentId: 'default', - }) - expect(c2.id).toBeDefined() - expect(c2.sessionId).toBe(sessionId) - }) - - it('should initialize a new session with chronological context, prefixed usernames, and resolved user mentions', async () => { - const team = await prisma.team.create({ data: { name: 't1' } }) - const user1 = await prisma.user.create({ - data: { name: 'User One', email: 'user1@example.com' }, - }) - const user2 = await prisma.user.create({ - data: { name: 'Matt', email: 'matt@example.com' }, - }) - const agentUser = await prisma.user.create({ - data: { id: 'agent-user-id', name: 'Smart Agent', email: 'agent@example.com', type: 'agent' }, - }) - const project = await prisma.project.create({ - data: { name: 'p1', teamId: team.id }, - }) - const file = await prisma.asset.create({ - data: { - name: 'test-file', - type: 'file', - projectId: project.id, - creatorId: user1.id, - status: 'uploaded', - }, - }) - - // Create existing comment 1 (User mentioning user2) - const comment1 = await prisma.assetComment.create({ - data: { - assetId: file.id, - creatorId: user1.id, - message: `First comment message for <@${user2.id}>`, - }, - }) - - // Create existing comment 1.5 (Agent) - const commentAgent = await prisma.assetComment.create({ - data: { - assetId: file.id, - creatorId: agentUser.id, - message: 'I am helping', - }, - }) - - // Create triggering comment 2 (Rule 1: no reply, mentions agent) - const comment2 = await prisma.assetComment.create({ - data: { - assetId: file.id, - creatorId: user1.id, - message: 'Hello <@agent-user-id>', - }, - }) - - const sessionId = await initializeAgentSessionActivity({ - teamId: team.id, - agentId: 'agent-user-id', - userCommentId: comment2.id, - userId: user1.id, - }) - - expect(sessionId).toBeDefined() - - // Verify session entries were populated with comment1 and commentAgent as context - const entries = await prisma.agentSessionEntry.findMany({ - where: { sessionId }, - orderBy: { id: 'asc' }, - }) - - expect(entries.length).toBe(2) - - // Verify User Comment Entry with resolved mention and username prefix - const entryData1 = entries[0].entry as unknown as SessionTreeEntry - expect(entryData1.id).toBeDefined() - expect(entryData1.id).not.toBe(comment1.id) - if (entryData1.type === 'message') { - const msg = entryData1.message as { role: 'user'; content: { type: 'text'; text: string }[] } - expect(msg.content[0].text).toBe('[User One]: First comment message for <@Matt>') - expect(msg.role).toBe('user') - } else { - throw new Error('Expected entry to be a message') - } - - // Verify Agent Comment Entry - const entryData2 = entries[1].entry as unknown as SessionTreeEntry - expect(entryData2.id).toBeDefined() - expect(entryData2.id).not.toBe(commentAgent.id) - if (entryData2.type === 'message') { - const msg = entryData2.message as { role: 'user'; content: { type: 'text'; text: string }[] } - expect(msg.content[0].text).toBe('[Agent Message][Smart Agent]: I am helping') - expect(msg.role).toBe('user') - } else { - throw new Error('Expected entry to be a message') - } - }) - - it('should support context-fetching and embedding database activities', async () => { - const team = await prisma.team.create({ data: { name: 't_ctx' } }) - const project = await prisma.project.create({ - data: { name: 'p_ctx', teamId: team.id }, - }) - const sk = await prisma.storageKey.create({ data: { key: 'asset-key' } }) - const asset = await prisma.asset.create({ - data: { - name: 'test-asset', - type: 'file', - projectId: project.id, - status: 'uploaded', - mediaType: 'image/png', - storageKeyId: sk.id, - }, - }) - - const provider = await prisma.provider.create({ - data: { - name: 'google', - teamId: team.id, - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- provider config requires broad any casting since it is defined as generic Json in Prisma schema - config: { api: 'google', apiKey: 'key' } as any, - }, - }) - - const model = await prisma.model.create({ - data: { - modelId: 'gemini', - name: 'Gemini', - providerId: provider.id, - config: { - reasoning: false, - input: ['text'], - contextWindow: 1000, - maxTokens: 1000, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- model config requires broad any casting since it is defined as generic Json in Prisma schema - } as any, - }, - }) - - const botUser = await prisma.user.create({ - data: { name: 'Pirate Bot', email: 'bot@example.com', type: 'agent' }, - }) - - const agent = await prisma.agent.create({ - data: { - id: botUser.id, - teamId: team.id, - type: 'chat', - enabled: true, - providerId: provider.id, - modelId: model.id, - config: { provider: 'google', model: 'gemini' }, - }, - }) - - await prisma.teamMember.create({ - data: { teamId: team.id, userId: botUser.id, role: 'reviewer' }, - }) - - // Test getAgentChatContextActivity - const chatCtx = await getAgentChatContextActivity({ teamId: team.id, agentId: agent.id }) - expect(chatCtx.agent.id).toBe(agent.id) - expect(chatCtx.dbProviders.length).toBe(1) - expect(chatCtx.allowedDomains).toEqual([]) - - // Test getAgentAutofillContextActivity (needs an autofill agent) - const autofillBot = await prisma.user.create({ - data: { name: 'Autofill Bot', email: 'autobot@example.com', type: 'agent' }, - }) - const autofillAgent = await prisma.agent.create({ - data: { - id: autofillBot.id, - teamId: team.id, - type: 'autofill', - enabled: true, - providerId: provider.id, - modelId: model.id, - config: { provider: 'google', model: 'gemini' }, - }, - }) - const autofillCtx = await getAgentAutofillContextActivity({ teamId: team.id }) - expect(autofillCtx.agent.id).toBe(autofillAgent.id) - - // Test getEmbeddingContextActivity (needs an embedding agent) - const embeddingBot = await prisma.user.create({ - data: { name: 'Embedding Bot', email: 'emb@example.com', type: 'agent' }, - }) - const embeddingAgent = await prisma.agent.create({ - data: { - id: embeddingBot.id, - teamId: team.id, - type: 'embedding', - enabled: true, - providerId: provider.id, - modelId: model.id, - config: { provider: 'google', model: 'gemini' }, - }, - }) - await prisma.teamMember.create({ - data: { teamId: team.id, userId: embeddingBot.id, role: 'reviewer' }, - }) - const embCtx = await getEmbeddingContextActivity({ teamId: team.id, assetId: asset.id }) - expect(embCtx.agent.id).toBe(embeddingAgent.id) - expect(embCtx.asset.id).toBe(asset.id) - - // Test saveAssetEmbeddingsActivity - const mockEmbedding = Array(1536).fill(0.1) - await saveAssetEmbeddingsActivity({ - assetId: asset.id, - embeddings: [{ embedding: mockEmbedding }], - }) - const dbEmbeddings = await prisma.assetEmbedding.findMany({ - where: { assetId: asset.id }, - }) - expect(dbEmbeddings.length).toBe(1) - }) - - describe('Agent System Tools and Path Context Activities', () => { - it('should correctly format getAssetPathContextActivity', async () => { - const team = await prisma.team.create({ data: { name: 'Path Team' } }) - const project = await prisma.project.create({ - data: { name: 'Path Project', teamId: team.id }, - }) - const folder1 = await prisma.asset.create({ - data: { - name: 'foo', - type: 'folder', - projectId: project.id, - status: 'uploaded', - }, - }) - const folder2 = await prisma.asset.create({ - data: { - name: 'bar', - type: 'folder', - projectId: project.id, - parentId: folder1.id, - status: 'uploaded', - }, - }) - const file = await prisma.asset.create({ - data: { - name: 'z.png', - type: 'file', - projectId: project.id, - parentId: folder2.id, - status: 'uploaded', - }, - }) - - const pathCtx = await getAssetPathContextActivity(file.id) - expect(pathCtx).toContain('Path: foo/bar/z.png') - expect(pathCtx).toContain(`name: foo, id: ${folder1.id}`) - expect(pathCtx).toContain(`name: bar, id: ${folder2.id}`) - expect(pathCtx).toContain(`name: z.png, id: ${file.id}`) - }) - - it('should correctly list assets, create folders, create files, and create versions', async () => { - const fs = await import('fs') - const path = await import('path') - - const team = await prisma.team.create({ data: { name: 'Tool Team' } }) - const user = await prisma.user.create({ - data: { name: 'Tool User', email: 'tooluser@example.com' }, - }) - await prisma.teamMember.create({ - data: { teamId: team.id, userId: user.id, role: 'owner' }, - }) - - const project = await prisma.project.create({ - data: { name: 'Tool Project', teamId: team.id }, - }) - - const parentFolder = await prisma.asset.create({ - data: { - name: 'Parent Folder', - type: 'folder', - projectId: project.id, - status: 'uploaded', - fileCount: 0, - }, - }) - - // 1. Test create_folder via executeAgentToolActivity - const folderResult = await executeAgentToolActivity({ - taskId: 'task1', - toolName: 'create_folder', - args: { - parent: parentFolder.id, - name: 'Child Folder', - }, - userId: user.id, - }) - expect(folderResult.id).toBeDefined() - expect(folderResult.name).toBe('Child Folder') - expect(folderResult.type).toBe('folder') - - const updatedParent = await prisma.asset.findUnique({ - where: { id: parentFolder.id }, - }) - expect(updatedParent?.fileCount).toBe(1) - - // 2. Test create_file via executeAgentToolActivity - const tempFilePath = path.join(process.cwd(), 'temp_test_file.txt') - fs.writeFileSync(tempFilePath, 'hello Shumai!') - - try { - const s3Key = await s3Service.uploadFile(tempFilePath, 'text/plain') - const fileResult = await executeAgentToolActivity({ - taskId: 'task2', - toolName: 'create_file', - args: { - parent: parentFolder.id, - s3Key, - name: 'temp_test_file.txt', - size: fs.statSync(tempFilePath).size, - contentType: 'text/plain', - }, - userId: user.id, - }) - expect(fileResult.id).toBeDefined() - expect(fileResult.name).toBe('temp_test_file.txt') - expect(fileResult.type).toBe('file') - - const parentAfterFile = await prisma.asset.findUnique({ - where: { id: parentFolder.id }, - }) - expect(parentAfterFile?.fileCount).toBe(2) - - // 3. Test list_assets via executeAgentToolActivity - const listResult = await executeAgentToolActivity({ - taskId: 'task3', - toolName: 'list_assets', - args: { - parent: parentFolder.id, - page: 1, - pageSize: 10, - type: 'all', - }, - userId: user.id, - }) - expect(listResult.assets).toBeDefined() - expect(listResult.assets.length).toBe(2) // folder and file - - // 4. Test create_version via executeAgentToolActivity (regular file -> stack) - const versionTempPath = path.join(process.cwd(), 'temp_version_file.txt') - fs.writeFileSync(versionTempPath, 'hello Shumai V2!') - - try { - const s3KeyV2 = await s3Service.uploadFile(versionTempPath, 'text/plain') - const versionResult = await executeAgentToolActivity({ - taskId: 'task4', - toolName: 'create_version', - args: { - parent: fileResult.id, - s3Key: s3KeyV2, - name: 'temp_version_file.txt', - size: fs.statSync(versionTempPath).size, - contentType: 'text/plain', - }, - userId: user.id, - }) - expect(versionResult.id).toBeDefined() - expect(versionResult.type).toBe('file') - - // Check if parent (the original file) was stacked - const originalFile = await prisma.asset.findUnique({ - where: { id: fileResult.id }, - include: { parent: true }, - }) - expect(originalFile?.parentId).toBeDefined() - expect(originalFile?.parent?.type).toBe('version_stack') - } finally { - if (fs.existsSync(versionTempPath)) { - fs.unlinkSync(versionTempPath) - } - } - } finally { - if (fs.existsSync(tempFilePath)) { - fs.unlinkSync(tempFilePath) - } - } - }) - - it('should trigger post-upload workflows (transcode, AI) when creating file or version via agent tool', async () => { - const team = await prisma.team.create({ data: { name: 'Workflow Team' } }) - const user = await prisma.user.create({ - data: { name: 'Workflow User', email: 'wfuser@example.com' }, - }) - await prisma.teamMember.create({ - data: { teamId: team.id, userId: user.id, role: 'owner' }, - }) - const project = await prisma.project.create({ - data: { name: 'Workflow Project', teamId: team.id }, - }) - const parentFolder = await prisma.asset.create({ - data: { - name: 'Parent Folder', - type: 'folder', - projectId: project.id, - status: 'uploaded', - }, - }) - - // Setup agents - const bot1 = await prisma.user.create({ - data: { name: 'Autofill Bot', email: 'autobot@example.com', type: 'agent' }, - }) - await prisma.teamMember.create({ - data: { teamId: team.id, userId: bot1.id, role: 'editor' }, - }) - await prisma.agent.create({ - data: { - user: { connect: { id: bot1.id } }, - type: 'autofill', - enabled: true, - team: { connect: { id: team.id } }, - config: { provider: 'google', model: 'gemini' }, - }, - }) - - const bot2 = await prisma.user.create({ - data: { name: 'Embedding Bot', email: 'emb@example.com', type: 'agent' }, - }) - await prisma.teamMember.create({ - data: { teamId: team.id, userId: bot2.id, role: 'editor' }, - }) - await prisma.agent.create({ - data: { - user: { connect: { id: bot2.id } }, - type: 'embedding', - enabled: true, - team: { connect: { id: team.id } }, - config: { provider: 'google', model: 'gemini' }, - }, - }) - - // Execute create_file with a video file type - const fileResult = await executeAgentToolActivity({ - taskId: 'wf-task-1', - toolName: 'create_file', - args: { - parent: parentFolder.id, - s3Key: 'video/some-key.mp4', - name: 'test-video.mp4', - size: 1024 * 1024, - contentType: 'video/mp4', - }, - userId: user.id, - }) - - expect(fileResult.id).toBeDefined() - - // Fetch WorkflowTasks created for the asset - const tasks = await prisma.workflowTask.findMany({ - where: { assetId: fileResult.id }, - }) - - // Assert post-upload workflows are triggered - expect(tasks.length).toBe(3) - const types = tasks.map((t) => t.type) - expect(types).toContain('transcode') - expect(types).toContain('ai_metadata_autofill') - expect(types).toContain('ai_embedding') - }) - - it('should assign a lower sortIndex to newer versions so they are correctly marked as latest', async () => { - const team = await prisma.team.create({ data: { name: 'Version Team' } }) - const user = await prisma.user.create({ - data: { name: 'Version User', email: 'vuser@example.com' }, - }) - await prisma.teamMember.create({ - data: { teamId: team.id, userId: user.id, role: 'owner' }, - }) - const project = await prisma.project.create({ - data: { name: 'Version Project', teamId: team.id }, - }) - const parentFolder = await prisma.asset.create({ - data: { - name: 'Parent Folder', - type: 'folder', - projectId: project.id, - status: 'uploaded', - }, - }) - - // Create file1 (V1) - const file1 = await prisma.asset.create({ - data: { - name: 'file1.txt', - type: 'file', - parentId: parentFolder.id, - projectId: project.id, - status: 'uploaded', - sizeByte: 100, - }, - }) - - // 1. Create file2 (V2) - Creates a new stack - const file2Result = await executeAgentToolActivity({ - taskId: 'vstack-task-1', - toolName: 'create_version', - args: { - parent: file1.id, - s3Key: 'file/file2-key/raw', - name: 'file2.txt', - size: 150, - contentType: 'text/plain', - }, - userId: user.id, - }) - - // Fetch from DB to check sortIndex - const updatedFile1 = await prisma.asset.findUnique({ where: { id: file1.id } }) - const updatedFile2 = await prisma.asset.findUnique({ where: { id: file2Result.id } }) - - // Newest version (file2) must have a strictly lower sortIndex than older version (file1) - expect(updatedFile2!.sortIndex! < updatedFile1!.sortIndex!).toBe(true) - - // 2. Create file3 (V3) - Appends to existing stack - const file3Result = await executeAgentToolActivity({ - taskId: 'vstack-task-2', - toolName: 'create_version', - args: { - parent: file2Result.id, - s3Key: 'file/file3-key/raw', - name: 'file3.txt', - size: 200, - contentType: 'text/plain', - }, - userId: user.id, - }) - - const updatedFile3 = await prisma.asset.findUnique({ where: { id: file3Result.id } }) - const freshFile2 = await prisma.asset.findUnique({ where: { id: file2Result.id } }) - - // Newest version (file3) must have a strictly lower sortIndex than version 2 (file2) - expect(updatedFile3!.sortIndex! < freshFile2!.sortIndex!).toBe(true) - }) - }) -}) diff --git a/packages/worker-db/src/activities/db.ts b/packages/worker-db/src/activities/db.ts deleted file mode 100644 index 67c0c98b..00000000 --- a/packages/worker-db/src/activities/db.ts +++ /dev/null @@ -1,1025 +0,0 @@ -import { prisma } from '@shumai/db' -import { ApplicationFailure } from '@temporalio/activity' -import type { AgentMessage, SessionTreeEntry } from '@earendil-works/pi-agent-core' -import { ulid } from 'ulid' -import { metadataService } from '@shumai/core/src/metadata/metadata' -import { UpdateAssetMetadataRequest } from '@shumai/dtos' -import { WorkflowTaskStatus, AssetStatus, Prisma, AssetType } from '@shumai/db' -import type { AgentExecutionContext } from '@shumai/agent' -import { assetService } from '@shumai/core/src/asset/asset' -import { VersionStackService } from '@shumai/core/src/versionStack/versionStack' -import { uploadService } from '@shumai/core/src/upload/upload' -import { authzService, Permission, ResourceType } from '@shumai/core/src/authz/authz' -import { generateKeyBetween } from 'jittered-fractional-indexing' -import { paginateQuery, encodeCursor } from '@shumai/core/src/pagination' - -export interface InitializeAgentSessionParams { - teamId: string - agentId: string - userCommentId: string - userId?: string -} - -export async function initializeAgentSessionActivity( - params: InitializeAgentSessionParams, -): Promise { - const agentId = params.agentId === 'default' ? 'default' : params.agentId - - // Ensure user exists for agent - const userExists = await prisma.user.findUnique({ where: { id: agentId } }) - if (!userExists) { - await prisma.user.create({ - data: { - id: agentId, - name: 'Ai Agent', - email: `${agentId}@shumai.ai`, - type: 'agent', - }, - }) - } - - // Ensure agent exists - const agentExists = await prisma.agent.findUnique({ where: { id: agentId } }) - if (!agentExists) { - await prisma.agent.create({ - data: { - id: agentId, - teamId: params.teamId, - type: 'chat', - config: { - provider: 'openai', - model: 'gpt-4', - }, - }, - }) - } - - // Create a new AgentSession - const newSession = await prisma.agentSession.create({ - data: { - agentId, - userId: params.userId || null, - cwd: process.cwd(), - }, - }) - const sessionId = newSession.id - - // Fetch existing comments as context - let existingComments: Array<{ - id: string - message: string | null - createdAt: Date - creatorId: string | null - creator: { type: string; name: string } | null - sessionId: string | null - }> = [] - - const userComment = await prisma.assetComment.findUnique({ - where: { id: params.userCommentId }, - include: { creator: true }, - }) - - if (userComment) { - if (!userComment.replyToId) { - // Rule 1: outside of a reply, add all other comments on that asset - existingComments = await prisma.assetComment.findMany({ - where: { - assetId: userComment.assetId, - id: { not: userComment.id }, - }, - orderBy: { id: 'asc' }, - include: { creator: true }, - }) - } else { - // Rule 2: in a reply, add root comment + all other replies to it - const rootComment = await prisma.assetComment.findUnique({ - where: { id: userComment.replyToId }, - include: { creator: true }, - }) - if (rootComment) { - const replies = await prisma.assetComment.findMany({ - where: { - replyToId: rootComment.id, - id: { not: userComment.id }, - }, - orderBy: { id: 'asc' }, - include: { creator: true }, - }) - existingComments = [rootComment, ...replies] - } - } - } - - // Resolve user mentions from IDs to names - const mentionRegex = /<@([^>]+)>/g - const mentionedUserIds = new Set() - for (const c of existingComments) { - if (c.message) { - const matches = [...c.message.matchAll(mentionRegex)] - for (const match of matches) { - mentionedUserIds.add(match[1]) - } - } - } - - const userIdToNameMap = new Map() - if (mentionedUserIds.size > 0) { - const resolvedUsers = await prisma.user.findMany({ - where: { - id: { in: Array.from(mentionedUserIds) }, - }, - select: { - id: true, - name: true, - }, - }) - for (const u of resolvedUsers) { - if (u.name) { - userIdToNameMap.set(u.id, u.name) - } - } - } - - // Save context as AgentSessionEntry records - let prevId: string | null = null - for (const c of existingComments) { - const isAgent = c.creator?.type === 'agent' || c.sessionId !== null - let messageContent = c.message || '' - - // Replace <@USER_ID> with <@USER_NAME> - messageContent = messageContent.replace(mentionRegex, (match, userId) => { - const resolvedName = userIdToNameMap.get(userId) - return resolvedName ? `<@${resolvedName}>` : match - }) - - if (isAgent) { - const agentName = c.creator?.name || 'Ai Agent' - messageContent = `[Agent Message][${agentName}]: ${messageContent}` - } else if (c.creator?.name) { - messageContent = `[${c.creator.name}]: ${messageContent}` - } - - const entryId = ulid() - const message: AgentMessage = { - role: 'user', - content: [{ type: 'text', text: messageContent }], - timestamp: c.createdAt.getTime(), - } - - const entryJson: SessionTreeEntry = { - type: 'message', - id: entryId, - parentId: prevId, - timestamp: c.createdAt.toISOString(), - message, - } - - await prisma.agentSessionEntry.create({ - data: { - id: entryId, - sessionId, - entry: entryJson, - }, - }) - prevId = entryId - } - - // Update leafId of the session - if (prevId) { - await prisma.agentSession.update({ - where: { id: sessionId }, - data: { leafId: prevId }, - }) - } - - return sessionId -} - -export async function deleteCommentActivity(commentId: string) { - return prisma.assetComment.delete({ - where: { id: commentId }, - }) -} - -export interface CreateCommentParams { - assetId: string - message: string - sessionId?: string | null - agentId?: string | null - replyToId?: string | null -} - -export async function createCommentActivity(params: CreateCommentParams) { - if (params.sessionId) { - const agentId = params.agentId && params.agentId !== 'default' ? params.agentId : 'default' - - // Fetch teamId from asset - const asset = await prisma.asset.findUnique({ - where: { id: params.assetId }, - include: { project: true }, - }) - if (!asset || !asset.project) { - throw ApplicationFailure.create({ message: 'asset or project not found', nonRetryable: true }) - } - const teamId = asset.project.teamId - - // Ensure user exists for agent - const userExists = await prisma.user.findUnique({ where: { id: agentId } }) - if (!userExists) { - await prisma.user.create({ - data: { - id: agentId, - name: 'Ai Agent', - email: `${agentId}@shumai.ai`, - type: 'agent', - }, - }) - } - - // Ensure agent exists - const agentExists = await prisma.agent.findUnique({ where: { id: agentId } }) - if (!agentExists) { - await prisma.agent.create({ - data: { - id: agentId, - teamId, - type: 'chat', - config: { - provider: 'openai', - model: 'gpt-4', - }, - }, - }) - } - - // Ensure agent session exists - const sessionExists = await prisma.agentSession.findUnique({ where: { id: params.sessionId } }) - if (!sessionExists) { - await prisma.agentSession.create({ - data: { - id: params.sessionId, - agentId: agentId, - cwd: process.cwd(), - }, - }) - } - } - - return prisma.assetComment.create({ - data: { - assetId: params.assetId, - message: params.message, - sessionId: params.sessionId ?? null, - creatorId: params.agentId && params.agentId !== 'default' ? params.agentId : null, - replyToId: params.replyToId, - }, - }) -} - -export interface UpdateCommentParams { - commentId: string - message: string - sessionId?: string | null -} - -export async function updateCommentActivity(params: UpdateCommentParams) { - return prisma.assetComment.update({ - where: { id: params.commentId }, - data: { - message: params.message, - ...(params.sessionId !== undefined ? { sessionId: params.sessionId } : {}), - }, - }) -} - -// ========================================== -// New Context-Fetching and Embedding DB Activities -// ========================================== - -export interface GetAgentChatContextParams { - teamId: string - agentId: string -} - -export async function getAgentChatContextActivity( - params: GetAgentChatContextParams, -): Promise { - const team = await prisma.team.findUnique({ - where: { id: params.teamId }, - }) - if (!team) { - throw ApplicationFailure.create({ message: 'failed to get team', nonRetryable: true }) - } - - const agent = await prisma.agent.findUnique({ - where: { id: params.agentId }, - include: { - provider: true, - modelRef: true, - }, - }) - if (!agent) { - throw ApplicationFailure.create({ - message: `agent ${params.agentId} not found`, - nonRetryable: true, - }) - } - - if (!agent.provider) { - throw ApplicationFailure.create({ - message: 'agent has no provider configured', - nonRetryable: true, - }) - } - if (!agent.modelRef) { - throw ApplicationFailure.create({ - message: 'agent has no model configured', - nonRetryable: true, - }) - } - - const providerName = agent.provider.name - - // Fetch the required provider configuration from database - const dbProviders = await prisma.provider.findMany({ - where: { teamId: params.teamId, name: providerName }, - include: { models: true }, - }) - - // Fetch team skills - const teamSkills = await prisma.skill.findMany({ - where: { teamId: params.teamId }, - }) - - const sandbox = await prisma.sandbox.findUnique({ - where: { teamId: params.teamId }, - }) - const allowedDomains = sandbox?.allowedDomains || [] - - return { - agent, - dbProviders, - teamSkills, - allowedDomains, - } -} - -export interface GetAgentAutofillContextParams { - teamId: string -} - -export async function getAgentAutofillContextActivity( - params: GetAgentAutofillContextParams, -): Promise { - const agent = await prisma.agent.findFirst({ - where: { - type: 'autofill', - enabled: true, - teamId: params.teamId, - }, - }) - if (!agent) { - throw ApplicationFailure.create({ - message: 'no autofill agent found for team', - nonRetryable: true, - }) - } - - const team = await prisma.team.findUnique({ - where: { id: params.teamId }, - }) - if (!team) { - throw ApplicationFailure.create({ message: 'failed to get team', nonRetryable: true }) - } - - // Find provider/model configuration - const agentWithDetails = await prisma.agent.findUnique({ - where: { id: agent.id }, - include: { - provider: true, - modelRef: true, - }, - }) - if (!agentWithDetails) { - throw ApplicationFailure.create({ message: `agent ${agent.id} not found`, nonRetryable: true }) - } - if (!agentWithDetails.provider) { - throw ApplicationFailure.create({ - message: 'agent has no provider configured', - nonRetryable: true, - }) - } - if (!agentWithDetails.modelRef) { - throw ApplicationFailure.create({ - message: 'agent has no model configured', - nonRetryable: true, - }) - } - - const providerName = agentWithDetails.provider.name - - // Fetch the required provider configuration from database - const dbProviders = await prisma.provider.findMany({ - where: { teamId: params.teamId, name: providerName }, - include: { models: true }, - }) - - // Fetch team skills - const teamSkills = await prisma.skill.findMany({ - where: { teamId: params.teamId }, - }) - - const sandbox = await prisma.sandbox.findUnique({ - where: { teamId: params.teamId }, - }) - const allowedDomains = sandbox?.allowedDomains || [] - - return { - agent: agentWithDetails, - dbProviders, - teamSkills, - allowedDomains, - } -} - -export interface GetEmbeddingContextParams { - teamId: string - assetId: string -} - -export async function getEmbeddingContextActivity(params: GetEmbeddingContextParams) { - const team = await prisma.team.findUnique({ - where: { id: params.teamId }, - }) - if (!team) { - throw ApplicationFailure.create({ message: 'failed to get team', nonRetryable: true }) - } - - const agent = await prisma.agent.findFirst({ - where: { - type: 'embedding', - enabled: true, - user: { teamMembers: { some: { teamId: params.teamId } } }, - }, - }) - if (!agent) { - throw ApplicationFailure.create({ - message: 'embedding feature is disabled or agent not found', - nonRetryable: true, - }) - } - - const asset = await prisma.asset.findUnique({ - where: { id: params.assetId }, - include: { storageKey: true }, - }) - if (!asset) { - throw ApplicationFailure.create({ message: 'failed to get asset', nonRetryable: true }) - } - if (!asset.mediaType) { - throw ApplicationFailure.create({ message: 'asset has no media type', nonRetryable: true }) - } - - return { - agent, - asset, - } -} - -export interface SaveAssetEmbeddingsParams { - assetId: string - embeddings: Array<{ - embedding: number[] - startTime?: number - endTime?: number - }> -} - -export async function saveAssetEmbeddingsActivity(params: SaveAssetEmbeddingsParams) { - for (const item of params.embeddings) { - const embVec = JSON.stringify(item.embedding) - if (item.startTime !== undefined && item.endTime !== undefined) { - await prisma.$executeRaw` - INSERT INTO asset_embeddings (id, asset_id, embedding, start_time, end_time, updated_at) - VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, ${item.startTime}, ${item.endTime}, NOW()) - ` - } else { - await prisma.$executeRaw` - INSERT INTO asset_embeddings (id, asset_id, embedding, updated_at) - VALUES (gen_random_uuid()::text, ${params.assetId}, ${embVec}::vector, NOW()) - ` - } - } -} - -// ========================================== -// Moved Database Activities -// ========================================== - -export async function getAssetActivity(assetId: string) { - return prisma.asset.findUnique({ - where: { id: assetId }, - include: { - storageKey: true, - project: { - include: { - team: true, - }, - }, - }, - }) -} - -export async function getCommentActivity(commentId: string) { - return prisma.assetComment.findUnique({ - where: { id: commentId }, - include: { attachments: { include: { asset: { include: { storageKey: true } } } } }, - }) -} - -export async function getProjectAutofillFieldsActivity(projectId: string) { - const fields = await metadataService.listProjectFields('', projectId) - return fields.filter((f) => f.field.aiAutofill).map((f) => f.field) -} - -export interface UpdateAssetMetadataParams { - assetId: string - metadata: UpdateAssetMetadataRequest[] -} - -export async function updateAssetMetadataActivity(params: UpdateAssetMetadataParams) { - return metadataService.updateAssetMetadata(params.assetId, params.metadata) -} - -export interface UpdateTaskStatusParams { - taskId: string - status: WorkflowTaskStatus - output?: unknown -} - -export async function updateTaskStatusActivity(params: UpdateTaskStatusParams): Promise { - await prisma.workflowTask.update({ - where: { id: params.taskId }, - data: { - status: params.status, - output: params.output, - }, - }) -} - -export interface UpdateAssetStatusParams { - assetId: string - status: AssetStatus -} - -export async function updateAssetStatusActivity(params: UpdateAssetStatusParams): Promise { - await prisma.asset.update({ - where: { id: params.assetId }, - data: { status: params.status }, - }) -} - -export interface UpdateTaskUsageParams { - taskId: string - inputTokens: number - outputTokens: number - model: string -} - -export async function updateTaskUsageActivity(params: UpdateTaskUsageParams): Promise { - await prisma.workflowTask.update({ - where: { id: params.taskId }, - data: { - inputTokens: params.inputTokens, - outputTokens: params.outputTokens, - model: params.model, - }, - }) -} - -export interface UpdateWorkflowTaskParams { - taskId: string - status?: WorkflowTaskStatus - heartbeat?: boolean - output?: unknown - inputTokens?: number - outputTokens?: number - model?: string | null -} - -export async function updateWorkflowTaskActivity(params: UpdateWorkflowTaskParams): Promise { - await prisma.workflowTask.update({ - where: { id: params.taskId }, - data: { - ...(params.status !== undefined ? { status: params.status } : {}), - ...(params.heartbeat ? { heartbeat: new Date() } : {}), - ...(params.output !== undefined ? { output: params.output as Prisma.InputJsonValue } : {}), - ...(params.inputTokens !== undefined ? { inputTokens: params.inputTokens } : {}), - ...(params.outputTokens !== undefined ? { outputTokens: params.outputTokens } : {}), - ...(params.model !== undefined ? { model: params.model } : {}), - }, - }) -} - -// ========================================== -// Agent System Tools & Context Activities -// ========================================== - -export async function getAssetPathContextActivity(assetId: string): Promise { - const parts: { name: string; id: string }[] = [] - let currentId: string | null = assetId - - while (currentId) { - /* prettier-ignore */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const assetNode: { id: string; name: string; parentId: string | null; type: AssetType } | null = (await prisma.asset.findUnique({ where: { id: currentId }, select: { id: true, name: true, parentId: true, type: true } })) as any - if (!assetNode) break - - if (assetNode.name) { - parts.unshift({ name: assetNode.name, id: assetNode.id }) - } - - currentId = assetNode.parentId - } - - if (parts.length === 0) return '' - - const pathStr = parts.map((p) => p.name).join('/') - let contextStr = `Path: ${pathStr}\n\n` - for (const part of parts) { - contextStr += `name: ${part.name}, id: ${part.id}\n` - } - - return contextStr -} - -export interface ExecuteAgentToolParams { - taskId: string - toolName: string - // args are dynamic JSON arguments passed from the agent harness at runtime - // eslint-disable-next-line @typescript-eslint/no-explicit-any - args: any - userId?: string -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export async function executeAgentToolActivity(params: ExecuteAgentToolParams): Promise { - const { toolName, args, userId } = params - if (!userId) { - throw ApplicationFailure.create({ - message: 'User ID is required for tool authz', - nonRetryable: true, - }) - } - - // 1. Fetch User - const user = await prisma.user.findUnique({ where: { id: userId } }) - if (!user) { - throw ApplicationFailure.create({ - message: `User not found: ${userId}`, - nonRetryable: true, - }) - } - - // 2. Perform Authz and Action based on toolName - switch (toolName) { - case 'list_assets': { - const parent = args.parent - if (!parent) { - throw ApplicationFailure.create({ - message: 'parent parameter is required', - nonRetryable: true, - }) - } - - // Authz check - await authzService.hasPermission({ - user, - permission: Permission.Read, - type: ResourceType.Asset, - id: parent, - }) - - const page = args.page || 1 - const pageSize = args.pageSize || 20 - const type = args.type || 'all' - - // We need to list files and folders of parent dir - const where: Prisma.AssetWhereInput = { - parentId: parent, - isDeleted: false, - } - if (type === 'file') { - where.type = { in: ['file', 'version_stack'] } - } else if (type === 'folder') { - where.type = 'folder' - } else { - where.type = { in: ['file', 'folder', 'version_stack'] } - } - - const limit = pageSize - const offset = (page - 1) * limit - const after = offset > 0 ? encodeCursor(offset) : undefined - - const { data: assets, pageInfo } = await paginateQuery< - Prisma.AssetGetPayload> - >( - async (skip, take) => { - return prisma.asset.findMany({ - where, - orderBy: { id: 'desc' }, // sort desc per ULID default in sorting guidelines! - skip, - take, - }) - }, - async () => prisma.asset.count({ where }), - { first: limit, after, includeCount: true }, - ) - - // return asset id, asset name, asset type, asset size (sizeByte or size) - const results = assets.map((a) => ({ - id: a.id, - name: a.name, - type: a.type, - size: a.sizeByte, - })) - - return { - assets: results, - pageInfo, - } - } - - case 'create_folder': { - const parent = args.parent - const name = args.name - if (!parent || !name) { - throw ApplicationFailure.create({ - message: 'parent and name parameters are required', - nonRetryable: true, - }) - } - - // Authz check - await authzService.hasPermission({ - user, - permission: Permission.Edit, - type: ResourceType.Asset, - id: parent, - }) - - const newFolder = await assetService.createAsset({ - name, - parentId: parent, - type: 'folder', - creatorId: userId, - }) - - // Increment fileCount of parent folder - await prisma.asset.update({ - where: { id: parent }, - data: { fileCount: { increment: 1 } }, - }) - - return { - id: newFolder.id, - name: newFolder.name, - type: newFolder.type, - size: newFolder.sizeByte, - } - } - - case 'create_file': { - const parent = args.parent - const s3Key = args.s3Key as string - const name = args.name as string - const fileSize = args.size as number - const mimeType = args.contentType as string - if (!parent || !s3Key || !name || fileSize === undefined || !mimeType) { - throw ApplicationFailure.create({ - message: 'parent, s3Key, name, size, and contentType parameters are required', - nonRetryable: true, - }) - } - - // Authz check - await authzService.hasPermission({ - user, - permission: Permission.Edit, - type: ResourceType.Asset, - id: parent, - }) - - // Fetch parent folder to resolve projectId and teamId - const parentAsset = await prisma.asset.findUnique({ - where: { id: parent }, - include: { project: true }, - }) - if (!parentAsset) { - throw ApplicationFailure.create({ - message: `Parent folder not found with ID: ${parent}`, - nonRetryable: true, - }) - } - if (!parentAsset.projectId || !parentAsset.project?.teamId) { - throw ApplicationFailure.create({ - message: `Parent folder ${parent} has no project or team associated`, - nonRetryable: true, - }) - } - - // Create asset via assetService - const newFile = await assetService.createAsset({ - name, - type: 'file', - parentId: parent, - key: s3Key, - sizeByte: fileSize, - contentType: mimeType, - creatorId: userId, - }) - - // Increment fileCount of parent and update ancestor size - await prisma.$transaction(async (tx) => { - await tx.asset.update({ - where: { id: parent }, - data: { fileCount: { increment: 1 } }, - }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await assetService.updateAncestorsSize(tx as any, parent, fileSize) - - // Trigger post-upload transcode and AI workflows - await uploadService.triggerPostUploadWorkflows( - tx, - newFile.id, - parentAsset.project!.teamId, - parentAsset.projectId!, - ) - }) - - return { - id: newFile.id, - name: newFile.name, - type: newFile.type, - size: newFile.sizeByte, - } - } - - case 'create_version': { - const parent = args.parent // parent file id - const s3Key = args.s3Key as string - const name = args.name as string - const fileSize = args.size as number - const mimeType = args.contentType as string - if (!parent || !s3Key || !name || fileSize === undefined || !mimeType) { - throw ApplicationFailure.create({ - message: 'parent, s3Key, name, size, and contentType parameters are required', - nonRetryable: true, - }) - } - - // Fetch parent file and cast to any due to Prisma type resolution limits - /* prettier-ignore */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const parentFile = (await prisma.asset.findUnique({ where: { id: parent }, include: { parent: true, project: true } })) as any - if (!parentFile) { - throw ApplicationFailure.create({ - message: `Parent file not found with ID: ${parent}`, - nonRetryable: true, - }) - } - - if (!parentFile.projectId || !parentFile.project?.teamId) { - throw ApplicationFailure.create({ - message: `Parent file ${parent} has no project or team associated`, - nonRetryable: true, - }) - } - - // Authz check: verify Edit permission on the parent file (resource ID is parent file id) - await authzService.hasPermission({ - user, - permission: Permission.Edit, - type: ResourceType.Asset, - id: parent, - }) - - // If already in a version stack - if (parentFile.parentId && parentFile.parent?.type === AssetType.version_stack) { - const stackId = parentFile.parentId - - // Create the new file asset - const newFile = await assetService.createAsset({ - name, - type: 'file', - parentId: stackId, - key: s3Key, - sizeByte: fileSize, - contentType: mimeType, - creatorId: userId, - }) - - // Generate sort index for new version inside the stack (newest gets the lowest sortIndex) - const firstChild = await prisma.asset.findFirst({ - where: { - parentId: stackId, - NOT: { id: newFile.id }, - }, - orderBy: { sortIndex: 'asc' }, - }) - const newSortIndex = generateKeyBetween(null, firstChild?.sortIndex || null) - - await prisma.$transaction(async (tx) => { - // Assign sort index - await tx.asset.update({ - where: { id: newFile.id }, - data: { sortIndex: newSortIndex }, - }) - // Increment stack's fileCount and size - const updatedStack = await tx.asset.update({ - where: { id: stackId }, - data: { - fileCount: { increment: 1 }, - sizeByte: { increment: fileSize }, - }, - }) - // Update ancestors' size of stack's parent - if (updatedStack.parentId) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await assetService.updateAncestorsSize(tx as any, updatedStack.parentId, fileSize) - } - - // Trigger post-upload transcode and AI workflows - await uploadService.triggerPostUploadWorkflows( - tx, - newFile.id, - parentFile.project.teamId, - parentFile.projectId, - ) - }) - - return { - id: newFile.id, - name: newFile.name, - type: newFile.type, - size: newFile.sizeByte, - } - } else { - // Parent is a regular file. We must create a new version stack. - const folderParentId = parentFile.parentId - if (!folderParentId) { - throw ApplicationFailure.create({ - message: `Parent file ${parent} has no parent folder`, - nonRetryable: true, - }) - } - - // Create the new file asset under parent folder first (so that createVersionStack can run on them, - // which enforces both files must have same parent initially) - const newFile = await assetService.createAsset({ - name, - type: 'file', - parentId: folderParentId, - key: s3Key, - sizeByte: fileSize, - contentType: mimeType, - creatorId: userId, - }) - - await prisma.$transaction(async (tx) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const txVersionStackService = new VersionStackService(tx as any) - await txVersionStackService.createVersionStack({ - fileIds: [newFile.id, parentFile.id], - projectId: parentFile.projectId!, - creatorId: userId, - }) - // Update ancestors' size (add size of the new file version) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await assetService.updateAncestorsSize(tx as any, folderParentId!, fileSize) - - // Trigger post-upload transcode and AI workflows - await uploadService.triggerPostUploadWorkflows( - tx, - newFile.id, - parentFile.project.teamId, - parentFile.projectId!, - ) - }) - - return { - id: newFile.id, - name: newFile.name, - type: newFile.type, - size: newFile.sizeByte, - } - } - } - - default: - throw ApplicationFailure.create({ - message: `Unsupported tool name: ${toolName}`, - nonRetryable: true, - }) - } -} diff --git a/packages/worker-db/src/index.ts b/packages/worker-db/src/index.ts deleted file mode 100644 index b42351dd..00000000 --- a/packages/worker-db/src/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { registerActivities } from '@shumai/workflow-core' -import * as dbActivities from './activities/db' - -export function initDbWorkflows() { - registerActivities(dbActivities) -} - -export * from './activities/db' diff --git a/packages/worker-db/tsconfig.json b/packages/worker-db/tsconfig.json deleted file mode 100644 index fc4e681f..00000000 --- a/packages/worker-db/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "ignoreDeprecations": "6.0" - }, - "include": ["src/**/*"] -} diff --git a/packages/workflow-core/src/activities/task.test.ts b/packages/workflow-core/src/activities/task.test.ts index 51526047..fa698bb6 100644 --- a/packages/workflow-core/src/activities/task.test.ts +++ b/packages/workflow-core/src/activities/task.test.ts @@ -5,7 +5,7 @@ import { updateTaskStatusActivity, updateAssetStatusActivity, updateTaskUsageActivity, -} from '@shumai/worker-db' +} from './task' import { WorkflowTaskStatus, WorkflowTaskType, AssetStatus } from '@shumai/db' describe('Task Activities', () => { diff --git a/packages/workflow-core/src/activities/task.ts b/packages/workflow-core/src/activities/task.ts index f1fcce1f..919754e6 100644 --- a/packages/workflow-core/src/activities/task.ts +++ b/packages/workflow-core/src/activities/task.ts @@ -1,7 +1,98 @@ +import { prisma } from '@shumai/db' +import { WorkflowTaskStatus, AssetStatus, Prisma } from '@shumai/db' + export async function getTranscodeWorkerQueueActivity(): Promise { return 'transcode_queue' } -export async function getAgentWorkerQueueActivity(): Promise { - return 'agent_queue' +export async function getAssetStatusActivity(params: { assetId: string }): Promise { + const asset = await prisma.asset.findUnique({ + where: { id: params.assetId }, + select: { status: true }, + }) + return asset?.status || null +} + +export async function getAssetActivity(assetId: string) { + return prisma.asset.findUnique({ + where: { id: assetId }, + include: { + storageKey: true, + project: { + include: { + team: true, + }, + }, + }, + }) +} + +export interface UpdateTaskStatusParams { + taskId: string + status: WorkflowTaskStatus + output?: unknown +} + +export async function updateTaskStatusActivity(params: UpdateTaskStatusParams): Promise { + await prisma.workflowTask.update({ + where: { id: params.taskId }, + data: { + status: params.status, + output: params.output as Prisma.InputJsonValue, + }, + }) +} + +export interface UpdateAssetStatusParams { + assetId: string + status: AssetStatus +} + +export async function updateAssetStatusActivity(params: UpdateAssetStatusParams): Promise { + await prisma.asset.update({ + where: { id: params.assetId }, + data: { status: params.status }, + }) +} + +export interface UpdateTaskUsageParams { + taskId: string + inputTokens: number + outputTokens: number + model: string +} + +export async function updateTaskUsageActivity(params: UpdateTaskUsageParams): Promise { + await prisma.workflowTask.update({ + where: { id: params.taskId }, + data: { + inputTokens: params.inputTokens, + outputTokens: params.outputTokens, + model: params.model, + }, + }) +} + +export interface UpdateWorkflowTaskParams { + taskId: string + status?: WorkflowTaskStatus + heartbeat?: boolean + output?: unknown + inputTokens?: number + outputTokens?: number + model?: string | null +} + +export async function updateWorkflowTaskActivity(params: UpdateWorkflowTaskParams): Promise { + await prisma.workflowTask.update({ + where: { id: params.taskId }, + data: { + ...(params.status !== undefined ? { status: params.status } : {}), + ...(params.heartbeat ? { heartbeat: new Date() } : {}), + ...(params.output !== undefined ? { output: params.output as Prisma.InputJsonValue } : {}), + ...(params.inputTokens !== undefined ? { inputTokens: params.inputTokens } : {}), + ...(params.outputTokens !== undefined ? { outputTokens: params.outputTokens } : {}), + ...(params.model !== undefined ? { model: params.model } : {}), + }, + }) } diff --git a/packages/workflow-core/src/temporal-executor.ts b/packages/workflow-core/src/temporal-executor.ts index 3cf83d8f..b2ae8ecc 100644 --- a/packages/workflow-core/src/temporal-executor.ts +++ b/packages/workflow-core/src/temporal-executor.ts @@ -1,8 +1,7 @@ import { Connection, Client } from '@temporalio/client' import { WorkflowTask, WorkflowTaskType } from '@shumai/db' import { Executor } from './executor' - -export const TASK_QUEUE_DB = 'db_queue' +import { TaskQueueAgent, TaskQueueTranscode } from './workflow-utils' export class TemporalExecutor implements Executor { private client: Client | null = null @@ -15,28 +14,35 @@ export class TemporalExecutor implements Executor { const workflowId = `${task.type}-${task.id}` let workflowName: string + let taskQueue: string + switch (task.type) { case WorkflowTaskType.ai_embedding: workflowName = 'agentEmbeddingWorkflow' + taskQueue = TaskQueueAgent break case WorkflowTaskType.query_embedding_for_search: workflowName = 'queryEmbeddingForSearch' + taskQueue = TaskQueueAgent break case WorkflowTaskType.ai_metadata_autofill: workflowName = 'agentAutofillWorkflow' + taskQueue = TaskQueueAgent break case WorkflowTaskType.chat: workflowName = 'agentChatWorkflow' + taskQueue = TaskQueueAgent break case WorkflowTaskType.transcode: workflowName = 'transcodeWorkflow' + taskQueue = TaskQueueTranscode break default: throw new Error(`Unknown task type: ${task.type}`) } const handle = await client.workflow.start(workflowName, { - taskQueue: TASK_QUEUE_DB, + taskQueue, workflowId, args: [task], }) diff --git a/packages/workflow-core/src/workflow-utils.ts b/packages/workflow-core/src/workflow-utils.ts index dd61acac..68ce9b02 100644 --- a/packages/workflow-core/src/workflow-utils.ts +++ b/packages/workflow-core/src/workflow-utils.ts @@ -4,7 +4,6 @@ import * as wf from '@temporalio/workflow' /** * Task queue constants. */ -export const TaskQueueDb = 'db_queue' export const TaskQueueAgent = 'agent_queue' export const TaskQueueTranscode = 'transcode_queue' diff --git a/packages/workflow-core/src/workflow.test.ts b/packages/workflow-core/src/workflow.test.ts index 16d471d3..f1c7dd18 100644 --- a/packages/workflow-core/src/workflow.test.ts +++ b/packages/workflow-core/src/workflow.test.ts @@ -4,7 +4,6 @@ import { workflowService } from './workflow' import { WorkflowTaskStatus, WorkflowTaskType } from '@shumai/db' import { initAgentWorkflows } from '@shumai/agent' import { initTranscodeWorkflows } from '@shumai/transcode' -import { initDbWorkflows } from '@shumai/worker-db' vi.mock('@shumai/core/src/s3/s3', () => ({ s3Service: { @@ -38,7 +37,6 @@ describe('WorkflowService', () => { beforeEach(async () => { initAgentWorkflows() initTranscodeWorkflows() - initDbWorkflows() await prisma.workflowTask.deleteMany() await prisma.assetComment.deleteMany() await prisma.asset.deleteMany() diff --git a/packages/workflow-core/src/workflow.ts b/packages/workflow-core/src/workflow.ts index 6da227fd..01355dc2 100644 --- a/packages/workflow-core/src/workflow.ts +++ b/packages/workflow-core/src/workflow.ts @@ -57,7 +57,7 @@ export class WorkflowService { // Starts Temporal workers for specific task queues async startWorkers( - queue: string = 'db_queue', + queue: string, options?: { // eslint-disable-next-line @typescript-eslint/no-explicit-any workflowBundle?: any @@ -77,18 +77,6 @@ export class WorkflowService { const activities = options?.activities || (globalThis as any).__localActivities const workflowBundle = options?.workflowBundle - if (queue === 'db_queue') { - console.log(`Starting Temporal worker for queue: ${queue}`) - const worker = await Worker.create({ - connection, - workflowBundle, - activities, - taskQueue: queue, - }) - await worker.run() - return - } - // For agent_queue and transcode_queue, use worker-specific unique queues const crypto = await import('crypto') const workerId = crypto.randomUUID() diff --git a/tsconfig.json b/tsconfig.json index 0e158f86..8d457c9d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,8 +21,7 @@ "@shumai/dtos": ["packages/dtos/src/index.ts"], "@shumai/workflow-core": ["packages/workflow-core/src/index.ts"], "@shumai/agent": ["packages/agent/src/index.ts"], - "@shumai/transcode": ["packages/transcode/src/index.ts"], - "@shumai/worker-db": ["packages/worker-db/src/index.ts"] + "@shumai/transcode": ["packages/transcode/src/index.ts"] } }, "exclude": ["node_modules"]