mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-20 01:16:54 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e223da8e3 | ||
|
|
92147c858f | ||
|
|
e602e96a28 | ||
|
|
cb1054bc2c | ||
|
|
87ccbe5330 | ||
|
|
04f0a39300 | ||
|
|
fd1ff8d70e | ||
|
|
79ea3c3ffd | ||
|
|
18780a9117 | ||
|
|
846732d7d4 | ||
|
|
1bf22231ec |
@@ -38,7 +38,7 @@ Here is a short demo video showing what the Shumai agent can do:
|
||||
|
||||
|
||||
|
||||
https://github.com/user-attachments/assets/2ebe5456-342e-47f7-9a93-a963fcf12097
|
||||
https://github.com/user-attachments/assets/1186067a-72c0-41b9-836f-987438f2332a
|
||||
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ Shumai 是一个开源的 Frame.io 替代方案,专为创意团队打造,让
|
||||
|
||||
下面是一个简短的 demo 视频,展示了 Shumai Agent 能做什么:
|
||||
|
||||
https://github.com/user-attachments/assets/2ebe5456-342e-47f7-9a93-a963fcf12097
|
||||
https://github.com/user-attachments/assets/1186067a-72c0-41b9-836f-987438f2332a
|
||||
|
||||
> [!NOTE]
|
||||
> **媒体处理与存储机制:** 上传媒体文件后,Shumai 始终会保留并存储原始文件。请注意,系统中显示的文件大小为原始文件的大小,不包含转码后生成的代理文件大小。无论图片还是视频,都会额外生成一张 480p 的 WebP 封面图,作为文件列表中的预览图。此外:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent-app",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/cli",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode-app",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"bin": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/web-app",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "shumai",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"repository": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/agent",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -92,7 +92,7 @@ async function executeAgentPrompt(params: {
|
||||
let systemPrompt = `You are the AI assistant for shumai.
|
||||
shumai is a professional creative collaboration platform similar to frame.io, where users can upload creative files, manage projects, assign tasks, get precise feedback, and share their work. As the agent, your role is to help users use and manage this platform.
|
||||
|
||||
shumai has its own cloud file system. If a user asks you to perform file system operations (for example: creating a folder, creating a file, stacking a version, or listing assets), you MUST use the corresponding agent system tools (e.g., 'create_folder', 'create_file', 'create_version', 'list_assets'). Do NOT use local bash commands or the local bash tool to perform these operations locally on the host environment; all operations must be executed through the platform's cloud file system tools so they are correctly registered and visible within the platform.
|
||||
shumai has its own cloud file system. If a user asks you to perform file system operations (for example: creating a folder, creating a file, stacking a version, or listing assets), you MUST use the corresponding agent system tools (e.g., 'create_folder', 'create_file', 'create_version', 'list_assets'). Before calling 'create_folder', first check if the folder exists using 'list_assets'. Do NOT use local bash commands or the local bash tool to perform these operations locally on the host environment; all operations must be executed through the platform's cloud file system tools so they are correctly registered and visible within the platform.
|
||||
|
||||
If you need to create files in the local filesystem (for example, a temporary file for uploading), only the '.pi' folder in the current directory has write permissions. Do NOT attempt to create files in any other directories.`
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { Type, type TSchema } from '@sinclair/typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { createAnalyzeImageTool } from './tools/analyze-image'
|
||||
@@ -153,7 +154,7 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
if (!fs.existsSync(piDir)) fs.mkdirSync(piDir, { recursive: true })
|
||||
|
||||
const allowWrite = [piDir]
|
||||
const allowWrite = [piDir, os.tmpdir()]
|
||||
|
||||
const sandboxState = {
|
||||
blockedHost: '',
|
||||
|
||||
@@ -137,32 +137,33 @@ describe('downloadAssetTool', () => {
|
||||
expect(result.details.size).toBe(Buffer.from('fake-video-bytes').length)
|
||||
})
|
||||
|
||||
it('should download proxy pdf to .pi directory when pdfTranscode is present', async () => {
|
||||
it('should download original file to .pi directory even when pdfTranscode is present', async () => {
|
||||
vi.mocked(authzService.hasPermission).mockResolvedValue()
|
||||
vi.mocked(prisma.asset.findUnique).mockResolvedValue({
|
||||
id: 'asset-pdf-1',
|
||||
name: 'doc.pdf',
|
||||
storageKey: { key: 'raw/doc.pdf' },
|
||||
id: 'asset-md-1',
|
||||
name: 'doc.md',
|
||||
storageKey: { key: 'raw/doc.md' },
|
||||
media: {
|
||||
proxyType: 'pdf',
|
||||
pdfTranscode: { key: 'proxy/doc.pdf' },
|
||||
original: { key: 'raw/doc.md' },
|
||||
},
|
||||
} as unknown as Asset)
|
||||
|
||||
vi.mocked(s3Service.getObject).mockResolvedValue({
|
||||
buffer: Buffer.from('fake-pdf-bytes'),
|
||||
contentType: 'application/pdf',
|
||||
buffer: Buffer.from('# fake markdown bytes'),
|
||||
contentType: 'text/markdown',
|
||||
} as unknown as { buffer: Buffer; contentType: string })
|
||||
|
||||
const tool = createDownloadAssetTool('user-1')
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-pdf-1' })
|
||||
const result = await tool.execute('call-1', { assetId: 'asset-md-1' })
|
||||
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'proxy/doc.pdf')
|
||||
expect(s3Service.getObject).toHaveBeenCalledWith('shumai', 'raw/doc.md')
|
||||
|
||||
const expectedPath = path.join(piDir, 'asset-pdf-1_doc.pdf')
|
||||
const expectedPath = path.join(piDir, 'asset-md-1_doc.md')
|
||||
createdFiles.push(expectedPath)
|
||||
|
||||
expect(fs.existsSync(expectedPath)).toBe(true)
|
||||
expect(result.details.contentType).toBe('application/pdf')
|
||||
expect(result.details.contentType).toBe('text/markdown')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -65,10 +65,6 @@ export function createDownloadAssetTool(userId: string): AgentTool<typeof downlo
|
||||
} else if (mediaInfo.videoPreview?.key) {
|
||||
mediaKey = mediaInfo.videoPreview.key
|
||||
}
|
||||
} else if (mediaInfo.proxyType === 'pdf' || mediaInfo.pdfTranscode?.key) {
|
||||
if (mediaInfo.pdfTranscode?.key) {
|
||||
mediaKey = mediaInfo.pdfTranscode.key
|
||||
}
|
||||
} else if (mediaInfo.original?.key) {
|
||||
mediaKey = mediaInfo.original.key || mediaKey
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { createSandboxedBashTool } from './sandboxed-bash'
|
||||
import { SandboxManager } from '@anthropic-ai/sandbox-runtime'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
|
||||
vi.mock('@anthropic-ai/sandbox-runtime')
|
||||
vi.mock('node:child_process')
|
||||
@@ -178,4 +179,105 @@ describe('createSandboxedBashTool', () => {
|
||||
await expect(executePromise).rejects.toThrow(/aborted/)
|
||||
killSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('should truncate output exceeding max lines (2000 lines) and persist full output to temp file', async () => {
|
||||
const tool = createSandboxedBashTool(mockCwd, mockSessionManager.getSkillEnvs())
|
||||
const mockStdout = new EventEmitter()
|
||||
const mockStderr = new EventEmitter()
|
||||
const mockChild = Object.assign(new EventEmitter(), {
|
||||
stdout: mockStdout,
|
||||
stderr: mockStderr,
|
||||
pid: 101,
|
||||
kill: vi.fn(),
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- spawn returns mocked ChildProcess
|
||||
;(spawn as any).mockReturnValue(mockChild)
|
||||
|
||||
const signal = createMockSignal()
|
||||
const executePromise = tool.execute('1', { command: 'generate lines' }, signal)
|
||||
|
||||
await vi.waitFor(() => expect(spawn).toHaveBeenCalled())
|
||||
|
||||
const lines: string[] = []
|
||||
for (let i = 1; i <= 3000; i++) {
|
||||
lines.push(`line-${i}`)
|
||||
}
|
||||
mockStdout.emit('data', Buffer.from(lines.join('\n') + '\n'))
|
||||
mockChild.emit('close', 0)
|
||||
|
||||
const result = await executePromise
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- content block array
|
||||
const text = (result as any).content[0].text
|
||||
expect(text).toContain('line-3000')
|
||||
expect(text).toContain('[Showing lines 1001-3000 of 3000. Full output:')
|
||||
expect(result.details?.truncation).toMatchObject({
|
||||
truncated: true,
|
||||
truncatedBy: 'lines',
|
||||
totalLines: 3000,
|
||||
outputLines: 2000,
|
||||
})
|
||||
|
||||
expect(result.details?.fullOutputPath).toBeDefined()
|
||||
const fileContent = await readFile(result.details!.fullOutputPath!, 'utf-8')
|
||||
expect(fileContent).toContain('line-1\nline-2')
|
||||
expect(fileContent).toContain('line-2999\nline-3000')
|
||||
})
|
||||
|
||||
it('should truncate output exceeding max bytes (50KB) and report partial line size', async () => {
|
||||
const tool = createSandboxedBashTool(mockCwd, mockSessionManager.getSkillEnvs())
|
||||
const mockStdout = new EventEmitter()
|
||||
const mockChild = Object.assign(new EventEmitter(), {
|
||||
stdout: mockStdout,
|
||||
stderr: new EventEmitter(),
|
||||
pid: 102,
|
||||
kill: vi.fn(),
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- spawn returns mocked ChildProcess
|
||||
;(spawn as any).mockReturnValue(mockChild)
|
||||
|
||||
const signal = createMockSignal()
|
||||
const executePromise = tool.execute('1', { command: 'long single line' }, signal)
|
||||
|
||||
await vi.waitFor(() => expect(spawn).toHaveBeenCalled())
|
||||
|
||||
const longLine = 'a'.repeat(60000)
|
||||
mockStdout.emit('data', Buffer.from(longLine))
|
||||
mockChild.emit('close', 0)
|
||||
|
||||
const result = await executePromise
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- content block array
|
||||
const text = (result as any).content[0].text
|
||||
expect(text).toMatch(/Showing last 50\.0KB of line 1 \(line is 58\.6KB\)\. Full output:/)
|
||||
})
|
||||
|
||||
it('should preserve truncated output when a command times out', async () => {
|
||||
vi.useFakeTimers()
|
||||
const tool = createSandboxedBashTool(mockCwd, mockSessionManager.getSkillEnvs())
|
||||
const mockStdout = new EventEmitter()
|
||||
const mockChild = Object.assign(new EventEmitter(), {
|
||||
stdout: mockStdout,
|
||||
stderr: new EventEmitter(),
|
||||
pid: 103,
|
||||
kill: vi.fn(),
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- spawn returns mocked ChildProcess
|
||||
;(spawn as any).mockReturnValue(mockChild)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- process.kill return type boolean
|
||||
const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true as any)
|
||||
|
||||
const signal = createMockSignal()
|
||||
const executePromise = tool.execute('1', { command: 'slow', timeout: 1 }, signal)
|
||||
|
||||
await vi.waitFor(() => expect(spawn).toHaveBeenCalled())
|
||||
|
||||
mockStdout.emit('data', Buffer.from('output before timeout\n'))
|
||||
await vi.advanceTimersByTimeAsync(1100)
|
||||
mockChild.emit('close', null)
|
||||
|
||||
await expect(executePromise).rejects.toThrow(/output before timeout/)
|
||||
await expect(executePromise).rejects.toThrow(/timed out after 1 seconds/)
|
||||
|
||||
vi.useRealTimers()
|
||||
killSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,16 +3,23 @@ import { SandboxManager } from '@anthropic-ai/sandbox-runtime'
|
||||
import { type AgentTool, type AgentToolResult } from '@earendil-works/pi-agent-core'
|
||||
import { Type } from '@sinclair/typebox'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { OutputAccumulator } from '../utils/output-accumulator'
|
||||
import {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
formatSize,
|
||||
type TruncationResult,
|
||||
} from '../utils/truncate'
|
||||
|
||||
const BashParameters = Type.Object({
|
||||
command: Type.String({ description: 'The bash command to execute.' }),
|
||||
timeout: Type.Optional(Type.Number({ description: 'Maximum execution time in seconds.' })),
|
||||
})
|
||||
|
||||
type BashDetails = {
|
||||
export interface BashDetails {
|
||||
exitCode: number | null
|
||||
stdout: string
|
||||
stderr: string
|
||||
truncation?: TruncationResult
|
||||
fullOutputPath?: string
|
||||
}
|
||||
|
||||
export interface SandboxedBashOptions {
|
||||
@@ -20,6 +27,8 @@ export interface SandboxedBashOptions {
|
||||
clearBlockedHost?: () => void
|
||||
}
|
||||
|
||||
const BASH_UPDATE_THROTTLE_MS = 100
|
||||
|
||||
export const createSandboxedBashTool = (
|
||||
cwd: string,
|
||||
skillEnvs: Record<string, string> = {},
|
||||
@@ -27,17 +36,106 @@ export const createSandboxedBashTool = (
|
||||
): AgentTool<typeof BashParameters, BashDetails> => {
|
||||
return {
|
||||
name: 'bash',
|
||||
description: 'Execute a bash command in a sandboxed environment.',
|
||||
description: `Execute a bash command in a sandboxed environment. Returns stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
|
||||
label: 'Executing bash command',
|
||||
parameters: BashParameters,
|
||||
execute: async (
|
||||
toolCallId,
|
||||
{ command, timeout },
|
||||
signal,
|
||||
onUpdate,
|
||||
): Promise<AgentToolResult<BashDetails>> => {
|
||||
options?.clearBlockedHost?.()
|
||||
const wrappedCommand = await SandboxManager.wrapWithSandbox(command)
|
||||
|
||||
const output = new OutputAccumulator({ tempFilePrefix: 'shumai-bash' })
|
||||
let acceptingOutput = true
|
||||
let updateTimer: NodeJS.Timeout | undefined
|
||||
let updateDirty = false
|
||||
let lastUpdateAt = 0
|
||||
|
||||
const emitOutputUpdate = () => {
|
||||
if (!onUpdate || !updateDirty) return
|
||||
updateDirty = false
|
||||
lastUpdateAt = Date.now()
|
||||
const snapshot = output.snapshot({ persistIfTruncated: true })
|
||||
onUpdate({
|
||||
content: [{ type: 'text', text: snapshot.content || '' }],
|
||||
details: {
|
||||
exitCode: null,
|
||||
truncation: snapshot.truncation.truncated ? snapshot.truncation : undefined,
|
||||
fullOutputPath: snapshot.fullOutputPath,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const clearUpdateTimer = () => {
|
||||
if (updateTimer) {
|
||||
clearTimeout(updateTimer)
|
||||
updateTimer = undefined
|
||||
}
|
||||
}
|
||||
|
||||
const scheduleOutputUpdate = () => {
|
||||
if (!onUpdate) return
|
||||
updateDirty = true
|
||||
const delay = BASH_UPDATE_THROTTLE_MS - (Date.now() - lastUpdateAt)
|
||||
if (delay <= 0) {
|
||||
clearUpdateTimer()
|
||||
emitOutputUpdate()
|
||||
return
|
||||
}
|
||||
updateTimer ??= setTimeout(() => {
|
||||
updateTimer = undefined
|
||||
emitOutputUpdate()
|
||||
}, delay)
|
||||
}
|
||||
|
||||
if (onUpdate) {
|
||||
onUpdate({ content: [], details: { exitCode: null } })
|
||||
}
|
||||
|
||||
const handleData = (data: Buffer) => {
|
||||
if (!acceptingOutput) return
|
||||
output.append(data)
|
||||
scheduleOutputUpdate()
|
||||
}
|
||||
|
||||
const finishOutput = async () => {
|
||||
acceptingOutput = false
|
||||
output.finish()
|
||||
clearUpdateTimer()
|
||||
emitOutputUpdate()
|
||||
const snapshot = output.snapshot({ persistIfTruncated: true })
|
||||
await output.closeTempFile()
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const formatOutput = (
|
||||
snapshot: Awaited<ReturnType<typeof finishOutput>>,
|
||||
emptyText = '(no output)',
|
||||
) => {
|
||||
const truncation = snapshot.truncation
|
||||
let text = snapshot.content || emptyText
|
||||
let details: BashDetails | undefined
|
||||
if (truncation.truncated) {
|
||||
details = { exitCode: null, truncation, fullOutputPath: snapshot.fullOutputPath }
|
||||
const startLine = truncation.totalLines - truncation.outputLines + 1
|
||||
const endLine = truncation.totalLines
|
||||
if (truncation.lastLinePartial) {
|
||||
const lastLineSize = formatSize(output.getLastLineBytes())
|
||||
text += `\n\n[Showing last ${formatSize(truncation.outputBytes)} of line ${endLine} (line is ${lastLineSize}). Full output: ${snapshot.fullOutputPath}]`
|
||||
} else if (truncation.truncatedBy === 'lines') {
|
||||
text += `\n\n[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}. Full output: ${snapshot.fullOutputPath}]`
|
||||
} else {
|
||||
text += `\n\n[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatSize(DEFAULT_MAX_BYTES)} limit). Full output: ${snapshot.fullOutputPath}]`
|
||||
}
|
||||
}
|
||||
return { text, details }
|
||||
}
|
||||
|
||||
const appendStatus = (text: string, status: string) => `${text ? `${text}\n\n` : ''}${status}`
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('bash', ['-c', wrappedCommand], {
|
||||
cwd,
|
||||
@@ -50,8 +148,6 @@ export const createSandboxedBashTool = (
|
||||
},
|
||||
})
|
||||
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let timedOut = false
|
||||
let timeoutHandle: NodeJS.Timeout | undefined
|
||||
|
||||
@@ -68,12 +164,8 @@ export const createSandboxedBashTool = (
|
||||
}, timeout * 1000)
|
||||
}
|
||||
|
||||
child.stdout?.on('data', (data) => {
|
||||
stdout += data.toString()
|
||||
})
|
||||
child.stderr?.on('data', (data) => {
|
||||
stderr += data.toString()
|
||||
})
|
||||
child.stdout?.on('data', handleData)
|
||||
child.stderr?.on('data', handleData)
|
||||
|
||||
const onAbort = () => {
|
||||
if (child.pid) {
|
||||
@@ -87,28 +179,38 @@ export const createSandboxedBashTool = (
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
|
||||
child.on('error', (err) => {
|
||||
child.on('error', async (err) => {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle)
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
const snapshot = await finishOutput()
|
||||
logger.error(
|
||||
{ err, command, stdout, stderr },
|
||||
{ err, command, content: snapshot.content },
|
||||
'Sandboxed bash command encountered process error',
|
||||
)
|
||||
reject(err)
|
||||
})
|
||||
|
||||
child.on('close', (code) => {
|
||||
child.on('close', async (code) => {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle)
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
|
||||
const blocked = options?.getBlockedHost?.()
|
||||
const snapshot = await finishOutput()
|
||||
const { text: outputText, details: truncationDetails } = formatOutput(snapshot)
|
||||
|
||||
if (signal?.aborted) {
|
||||
logger.error({ command, stdout, stderr }, 'Sandboxed bash command aborted')
|
||||
reject(new Error('aborted'))
|
||||
logger.error({ command, outputText }, 'Sandboxed bash command aborted')
|
||||
reject(new Error(appendStatus(outputText, 'Command aborted')))
|
||||
} else if (timedOut) {
|
||||
logger.error({ timeout, command, stdout, stderr }, 'Sandboxed bash command timed out')
|
||||
reject(new Error(`bash command timed out after ${timeout} seconds`))
|
||||
logger.error({ timeout, command, outputText }, 'Sandboxed bash command timed out')
|
||||
reject(
|
||||
new Error(
|
||||
appendStatus(
|
||||
outputText,
|
||||
`Sandboxed bash command timed out after ${timeout} seconds`,
|
||||
),
|
||||
),
|
||||
)
|
||||
} else if (blocked) {
|
||||
reject(
|
||||
new Error(
|
||||
@@ -117,23 +219,25 @@ export const createSandboxedBashTool = (
|
||||
)
|
||||
} else if (code !== 0 && code !== null) {
|
||||
logger.error(
|
||||
{ exitCode: code, command, stdout, stderr },
|
||||
{ exitCode: code, command, outputText },
|
||||
'Sandboxed bash command exited with failure code',
|
||||
)
|
||||
reject(new Error(`bash command exited with code ${code}`))
|
||||
reject(
|
||||
new Error(
|
||||
appendStatus(outputText, `Sandboxed bash command exited with code ${code}`),
|
||||
),
|
||||
)
|
||||
} else {
|
||||
const output = stdout + stderr
|
||||
resolve({
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: output || (code === 0 ? 'Success' : `Failed with exit code ${code}`),
|
||||
text: outputText,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
exitCode: code,
|
||||
stdout,
|
||||
stderr,
|
||||
...(truncationDetails ?? {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { createWriteStream, type WriteStream } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
type TruncationResult,
|
||||
truncateTail,
|
||||
} from './truncate'
|
||||
|
||||
export interface OutputAccumulatorOptions {
|
||||
maxLines?: number
|
||||
maxBytes?: number
|
||||
tempFilePrefix?: string
|
||||
}
|
||||
|
||||
export interface OutputSnapshot {
|
||||
content: string
|
||||
truncation: TruncationResult
|
||||
fullOutputPath?: string
|
||||
}
|
||||
|
||||
function defaultTempFilePath(prefix: string): string {
|
||||
const id = randomBytes(8).toString('hex')
|
||||
return join(tmpdir(), `${prefix}-${id}.log`)
|
||||
}
|
||||
|
||||
function byteLength(text: string): number {
|
||||
return Buffer.byteLength(text, 'utf-8')
|
||||
}
|
||||
|
||||
export class OutputAccumulator {
|
||||
private readonly maxLines: number
|
||||
private readonly maxBytes: number
|
||||
private readonly maxRollingBytes: number
|
||||
private readonly tempFilePrefix: string
|
||||
private readonly decoder = new TextDecoder()
|
||||
|
||||
private rawChunks: Buffer[] = []
|
||||
private tailText = ''
|
||||
private tailBytes = 0
|
||||
private tailStartsAtLineBoundary = true
|
||||
private totalRawBytes = 0
|
||||
private totalDecodedBytes = 0
|
||||
private completedLines = 0
|
||||
private totalLines = 0
|
||||
private currentLineBytes = 0
|
||||
private hasOpenLine = false
|
||||
private finished = false
|
||||
|
||||
private tempFilePath: string | undefined
|
||||
private tempFileStream: WriteStream | undefined
|
||||
|
||||
constructor(options: OutputAccumulatorOptions = {}) {
|
||||
this.maxLines = options.maxLines ?? DEFAULT_MAX_LINES
|
||||
this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
|
||||
this.maxRollingBytes = Math.max(this.maxBytes * 2, 1)
|
||||
this.tempFilePrefix = options.tempFilePrefix ?? 'shumai-bash'
|
||||
}
|
||||
|
||||
append(data: Buffer): void {
|
||||
if (this.finished) {
|
||||
throw new Error('Cannot append to a finished output accumulator')
|
||||
}
|
||||
|
||||
this.totalRawBytes += data.length
|
||||
this.appendDecodedText(this.decoder.decode(data, { stream: true }))
|
||||
|
||||
if (this.tempFileStream || this.shouldUseTempFile()) {
|
||||
this.ensureTempFile()
|
||||
this.tempFileStream?.write(data)
|
||||
} else if (data.length > 0) {
|
||||
this.rawChunks.push(data)
|
||||
}
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (this.finished) {
|
||||
return
|
||||
}
|
||||
this.finished = true
|
||||
this.appendDecodedText(this.decoder.decode())
|
||||
if (this.shouldUseTempFile()) {
|
||||
this.ensureTempFile()
|
||||
}
|
||||
}
|
||||
|
||||
snapshot(options: { persistIfTruncated?: boolean } = {}): OutputSnapshot {
|
||||
const tailTruncation = truncateTail(this.getSnapshotText(), {
|
||||
maxLines: this.maxLines,
|
||||
maxBytes: this.maxBytes,
|
||||
})
|
||||
const truncated = this.totalLines > this.maxLines || this.totalDecodedBytes > this.maxBytes
|
||||
const truncatedBy = truncated
|
||||
? (tailTruncation.truncatedBy ?? (this.totalDecodedBytes > this.maxBytes ? 'bytes' : 'lines'))
|
||||
: null
|
||||
const truncation: TruncationResult = {
|
||||
...tailTruncation,
|
||||
truncated,
|
||||
truncatedBy,
|
||||
totalLines: this.totalLines,
|
||||
totalBytes: this.totalDecodedBytes,
|
||||
maxLines: this.maxLines,
|
||||
maxBytes: this.maxBytes,
|
||||
}
|
||||
|
||||
if (options.persistIfTruncated && truncation.truncated) {
|
||||
this.ensureTempFile()
|
||||
}
|
||||
|
||||
return {
|
||||
content: truncation.content,
|
||||
truncation,
|
||||
fullOutputPath: this.tempFilePath,
|
||||
}
|
||||
}
|
||||
|
||||
async closeTempFile(): Promise<void> {
|
||||
if (!this.tempFileStream) {
|
||||
return
|
||||
}
|
||||
|
||||
const stream = this.tempFileStream
|
||||
this.tempFileStream = undefined
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onError = (error: Error) => {
|
||||
stream.off('finish', onFinish)
|
||||
reject(error)
|
||||
}
|
||||
const onFinish = () => {
|
||||
stream.off('error', onError)
|
||||
resolve()
|
||||
}
|
||||
stream.once('error', onError)
|
||||
stream.once('finish', onFinish)
|
||||
stream.end()
|
||||
})
|
||||
}
|
||||
|
||||
getLastLineBytes(): number {
|
||||
return this.currentLineBytes
|
||||
}
|
||||
|
||||
private appendDecodedText(text: string): void {
|
||||
if (text.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const bytes = byteLength(text)
|
||||
this.totalDecodedBytes += bytes
|
||||
this.tailText += text
|
||||
this.tailBytes += bytes
|
||||
if (this.tailBytes > this.maxRollingBytes * 2) {
|
||||
this.trimTail()
|
||||
}
|
||||
|
||||
let newlines = 0
|
||||
let lastNewline = -1
|
||||
for (let i = text.indexOf('\n'); i !== -1; i = text.indexOf('\n', i + 1)) {
|
||||
newlines++
|
||||
lastNewline = i
|
||||
}
|
||||
if (newlines === 0) {
|
||||
this.currentLineBytes += bytes
|
||||
this.hasOpenLine = true
|
||||
} else {
|
||||
this.completedLines += newlines
|
||||
const tail = text.slice(lastNewline + 1)
|
||||
this.currentLineBytes = byteLength(tail)
|
||||
this.hasOpenLine = tail.length > 0
|
||||
}
|
||||
this.totalLines = this.completedLines + (this.hasOpenLine ? 1 : 0)
|
||||
}
|
||||
|
||||
private trimTail(): void {
|
||||
const buffer = Buffer.from(this.tailText, 'utf-8')
|
||||
if (buffer.length <= this.maxRollingBytes) {
|
||||
this.tailBytes = buffer.length
|
||||
return
|
||||
}
|
||||
|
||||
let start = buffer.length - this.maxRollingBytes
|
||||
while (start < buffer.length && (buffer[start] & 0xc0) === 0x80) {
|
||||
start++
|
||||
}
|
||||
|
||||
this.tailStartsAtLineBoundary =
|
||||
start === 0 ? this.tailStartsAtLineBoundary : buffer[start - 1] === 0x0a
|
||||
this.tailText = buffer.subarray(start).toString('utf-8')
|
||||
this.tailBytes = byteLength(this.tailText)
|
||||
}
|
||||
|
||||
private getSnapshotText(): string {
|
||||
if (this.tailStartsAtLineBoundary) {
|
||||
return this.tailText
|
||||
}
|
||||
|
||||
const firstNewline = this.tailText.indexOf('\n')
|
||||
return firstNewline === -1 ? this.tailText : this.tailText.slice(firstNewline + 1)
|
||||
}
|
||||
|
||||
private shouldUseTempFile(): boolean {
|
||||
return (
|
||||
this.totalRawBytes > this.maxBytes ||
|
||||
this.totalDecodedBytes > this.maxBytes ||
|
||||
this.totalLines > this.maxLines
|
||||
)
|
||||
}
|
||||
|
||||
private ensureTempFile(): void {
|
||||
if (this.tempFilePath) {
|
||||
return
|
||||
}
|
||||
this.tempFilePath = defaultTempFilePath(this.tempFilePrefix)
|
||||
this.tempFileStream = createWriteStream(this.tempFilePath)
|
||||
for (const chunk of this.rawChunks) {
|
||||
this.tempFileStream.write(chunk)
|
||||
}
|
||||
this.rawChunks = []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
export const DEFAULT_MAX_LINES = 2000
|
||||
export const DEFAULT_MAX_BYTES = 50 * 1024 // 50KB
|
||||
|
||||
export interface TruncationResult {
|
||||
content: string
|
||||
truncated: boolean
|
||||
truncatedBy: 'lines' | 'bytes' | null
|
||||
totalLines: number
|
||||
totalBytes: number
|
||||
outputLines: number
|
||||
outputBytes: number
|
||||
lastLinePartial: boolean
|
||||
firstLineExceedsLimit: boolean
|
||||
maxLines: number
|
||||
maxBytes: number
|
||||
}
|
||||
|
||||
export interface TruncationOptions {
|
||||
maxLines?: number
|
||||
maxBytes?: number
|
||||
}
|
||||
|
||||
export function formatSize(bytes: number): string {
|
||||
if (bytes < 1024) {
|
||||
return `${bytes}B`
|
||||
} else if (bytes < 1024 * 1024) {
|
||||
return `${(bytes / 1024).toFixed(1)}KB`
|
||||
} else {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`
|
||||
}
|
||||
}
|
||||
|
||||
function splitLinesForCounting(content: string): string[] {
|
||||
if (content.length === 0) {
|
||||
return []
|
||||
}
|
||||
const lines = content.split('\n')
|
||||
if (content.endsWith('\n')) {
|
||||
lines.pop()
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function truncateStringToBytesFromEnd(str: string, maxBytes: number): string {
|
||||
const buf = Buffer.from(str, 'utf-8')
|
||||
if (buf.length <= maxBytes) {
|
||||
return str
|
||||
}
|
||||
|
||||
let start = buf.length - maxBytes
|
||||
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
|
||||
start++
|
||||
}
|
||||
|
||||
return buf.subarray(start).toString('utf-8')
|
||||
}
|
||||
|
||||
export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
|
||||
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
|
||||
|
||||
const totalBytes = Buffer.byteLength(content, 'utf-8')
|
||||
const lines = splitLinesForCounting(content)
|
||||
const totalLines = lines.length
|
||||
|
||||
if (totalLines <= maxLines && totalBytes <= maxBytes) {
|
||||
return {
|
||||
content,
|
||||
truncated: false,
|
||||
truncatedBy: null,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: totalLines,
|
||||
outputBytes: totalBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
}
|
||||
}
|
||||
|
||||
const outputLinesArr: string[] = []
|
||||
let outputBytesCount = 0
|
||||
let truncatedBy: 'lines' | 'bytes' = 'lines'
|
||||
let lastLinePartial = false
|
||||
|
||||
for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) {
|
||||
const line = lines[i]
|
||||
const lineBytes = Buffer.byteLength(line, 'utf-8') + (outputLinesArr.length > 0 ? 1 : 0)
|
||||
|
||||
if (outputBytesCount + lineBytes > maxBytes) {
|
||||
truncatedBy = 'bytes'
|
||||
if (outputLinesArr.length === 0) {
|
||||
const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes)
|
||||
outputLinesArr.unshift(truncatedLine)
|
||||
outputBytesCount = Buffer.byteLength(truncatedLine, 'utf-8')
|
||||
lastLinePartial = true
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
outputLinesArr.unshift(line)
|
||||
outputBytesCount += lineBytes
|
||||
}
|
||||
|
||||
if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
|
||||
truncatedBy = 'lines'
|
||||
}
|
||||
|
||||
const outputContent = outputLinesArr.join('\n')
|
||||
const finalOutputBytes = Buffer.byteLength(outputContent, 'utf-8')
|
||||
|
||||
return {
|
||||
content: outputContent,
|
||||
truncated: true,
|
||||
truncatedBy,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: outputLinesArr.length,
|
||||
outputBytes: finalOutputBytes,
|
||||
lastLinePartial,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/api",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/core",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/db",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/dtos",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/e2e",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/tableprinter",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/transcode",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
Trash2,
|
||||
Wrench,
|
||||
} from 'lucide-react'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { useCallback, useEffect, useRef } from 'react'
|
||||
import Markdown from 'react-markdown'
|
||||
import { formatTimeAgo } from '../lib/time'
|
||||
|
||||
@@ -53,6 +53,8 @@ export function ChatbotSidebar({ projectId, contextAssetId }: ChatbotSidebarProp
|
||||
abortActiveSession,
|
||||
selectedAgentId,
|
||||
setSelectedAgentId,
|
||||
inputText,
|
||||
setInputText,
|
||||
} = useChatbotStore()
|
||||
|
||||
const { teamId, ensureTeamIdForProject } = useTeamContextStore()
|
||||
@@ -87,7 +89,6 @@ export function ChatbotSidebar({ projectId, contextAssetId }: ChatbotSidebarProp
|
||||
}
|
||||
}, [chatAgents, selectedAgentId, setSelectedAgentId])
|
||||
|
||||
const [inputText, setInputText] = useState('')
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null)
|
||||
const scrollAreaRef = useRef<HTMLDivElement>(null)
|
||||
const isFirstScrollRef = useRef(true)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/webui",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./index.html",
|
||||
|
||||
@@ -39,9 +39,26 @@ describe('useChatbotStore', () => {
|
||||
historySessions: [],
|
||||
isStreaming: false,
|
||||
selectedAgentId: 'agent-1',
|
||||
inputText: '',
|
||||
})
|
||||
})
|
||||
|
||||
it('should persist and reset inputText', () => {
|
||||
expect(useChatbotStore.getState().inputText).toBe('')
|
||||
useChatbotStore.getState().setInputText('Draft text in progress')
|
||||
expect(useChatbotStore.getState().inputText).toBe('Draft text in progress')
|
||||
|
||||
useChatbotStore.setState({
|
||||
currentSessionId: 'sess-abc',
|
||||
messages: [{ id: 'msg-1', role: 'user', content: 'test' } as unknown as ChatMessage],
|
||||
})
|
||||
// Navigating or changing other state does not wipe inputText
|
||||
expect(useChatbotStore.getState().inputText).toBe('Draft text in progress')
|
||||
|
||||
useChatbotStore.getState().startNewSession()
|
||||
expect(useChatbotStore.getState().inputText).toBe('')
|
||||
})
|
||||
|
||||
it('should add, remove and clear assets in context', () => {
|
||||
const store = useChatbotStore.getState()
|
||||
expect(store.chatAssets).toHaveLength(0)
|
||||
|
||||
@@ -28,6 +28,8 @@ interface ChatbotState {
|
||||
setIsStreaming: (streaming: boolean) => void
|
||||
selectedAgentId: string | null
|
||||
setSelectedAgentId: (id: string | null) => void
|
||||
inputText: string
|
||||
setInputText: (text: string) => void
|
||||
|
||||
scrollTop: number
|
||||
isAtBottom: boolean
|
||||
@@ -74,6 +76,8 @@ export const useChatbotStore = create<ChatbotState>((set) => ({
|
||||
setIsStreaming: (streaming) => set({ isStreaming: streaming }),
|
||||
selectedAgentId: null,
|
||||
setSelectedAgentId: (id) => set({ selectedAgentId: id }),
|
||||
inputText: '',
|
||||
setInputText: (text) => set({ inputText: text }),
|
||||
scrollTop: 0,
|
||||
isAtBottom: true,
|
||||
setScrollState: (scrollTop, isAtBottom) => set({ scrollTop, isAtBottom }),
|
||||
@@ -147,6 +151,7 @@ export const useChatbotStore = create<ChatbotState>((set) => ({
|
||||
currentSessionId: null,
|
||||
messages: [],
|
||||
chatAssets: [],
|
||||
inputText: '',
|
||||
scrollTop: 0,
|
||||
isAtBottom: true,
|
||||
})
|
||||
|
||||
@@ -1 +1 @@
|
||||
export const APP_VERSION = '0.3.1'
|
||||
export const APP_VERSION = '0.3.2'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@shumai/workflow-core",
|
||||
"version": "0.3.1",
|
||||
"version": "0.3.2",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
|
||||
Reference in New Issue
Block a user