mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 06:07:41 +00:00
Merge pull request #248 from shumaiOne/feat/sandboxed-bash-truncation
feat(agent): add output truncation and streaming limits to sandboxed bash tool
This commit is contained in:
@@ -11,6 +11,7 @@ import { agentService } from '@shumai/core/src/agent/agent'
|
||||
import { prisma } from '@shumai/db'
|
||||
import { Type, type TSchema } from '@sinclair/typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as os from 'os'
|
||||
import * as path from 'path'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { createAnalyzeImageTool } from './tools/analyze-image'
|
||||
@@ -153,7 +154,7 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
const piDir = path.join(process.cwd(), '.pi')
|
||||
if (!fs.existsSync(piDir)) fs.mkdirSync(piDir, { recursive: true })
|
||||
|
||||
const allowWrite = [piDir]
|
||||
const allowWrite = [piDir, os.tmpdir()]
|
||||
|
||||
const sandboxState = {
|
||||
blockedHost: '',
|
||||
|
||||
@@ -3,6 +3,7 @@ import { createSandboxedBashTool } from './sandboxed-bash'
|
||||
import { SandboxManager } from '@anthropic-ai/sandbox-runtime'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
|
||||
vi.mock('@anthropic-ai/sandbox-runtime')
|
||||
vi.mock('node:child_process')
|
||||
@@ -178,4 +179,105 @@ describe('createSandboxedBashTool', () => {
|
||||
await expect(executePromise).rejects.toThrow(/aborted/)
|
||||
killSpy.mockRestore()
|
||||
})
|
||||
|
||||
it('should truncate output exceeding max lines (2000 lines) and persist full output to temp file', async () => {
|
||||
const tool = createSandboxedBashTool(mockCwd, mockSessionManager.getSkillEnvs())
|
||||
const mockStdout = new EventEmitter()
|
||||
const mockStderr = new EventEmitter()
|
||||
const mockChild = Object.assign(new EventEmitter(), {
|
||||
stdout: mockStdout,
|
||||
stderr: mockStderr,
|
||||
pid: 101,
|
||||
kill: vi.fn(),
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- spawn returns mocked ChildProcess
|
||||
;(spawn as any).mockReturnValue(mockChild)
|
||||
|
||||
const signal = createMockSignal()
|
||||
const executePromise = tool.execute('1', { command: 'generate lines' }, signal)
|
||||
|
||||
await vi.waitFor(() => expect(spawn).toHaveBeenCalled())
|
||||
|
||||
const lines: string[] = []
|
||||
for (let i = 1; i <= 3000; i++) {
|
||||
lines.push(`line-${i}`)
|
||||
}
|
||||
mockStdout.emit('data', Buffer.from(lines.join('\n') + '\n'))
|
||||
mockChild.emit('close', 0)
|
||||
|
||||
const result = await executePromise
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- content block array
|
||||
const text = (result as any).content[0].text
|
||||
expect(text).toContain('line-3000')
|
||||
expect(text).toContain('[Showing lines 1001-3000 of 3000. Full output:')
|
||||
expect(result.details?.truncation).toMatchObject({
|
||||
truncated: true,
|
||||
truncatedBy: 'lines',
|
||||
totalLines: 3000,
|
||||
outputLines: 2000,
|
||||
})
|
||||
|
||||
expect(result.details?.fullOutputPath).toBeDefined()
|
||||
const fileContent = await readFile(result.details!.fullOutputPath!, 'utf-8')
|
||||
expect(fileContent).toContain('line-1\nline-2')
|
||||
expect(fileContent).toContain('line-2999\nline-3000')
|
||||
})
|
||||
|
||||
it('should truncate output exceeding max bytes (50KB) and report partial line size', async () => {
|
||||
const tool = createSandboxedBashTool(mockCwd, mockSessionManager.getSkillEnvs())
|
||||
const mockStdout = new EventEmitter()
|
||||
const mockChild = Object.assign(new EventEmitter(), {
|
||||
stdout: mockStdout,
|
||||
stderr: new EventEmitter(),
|
||||
pid: 102,
|
||||
kill: vi.fn(),
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- spawn returns mocked ChildProcess
|
||||
;(spawn as any).mockReturnValue(mockChild)
|
||||
|
||||
const signal = createMockSignal()
|
||||
const executePromise = tool.execute('1', { command: 'long single line' }, signal)
|
||||
|
||||
await vi.waitFor(() => expect(spawn).toHaveBeenCalled())
|
||||
|
||||
const longLine = 'a'.repeat(60000)
|
||||
mockStdout.emit('data', Buffer.from(longLine))
|
||||
mockChild.emit('close', 0)
|
||||
|
||||
const result = await executePromise
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- content block array
|
||||
const text = (result as any).content[0].text
|
||||
expect(text).toMatch(/Showing last 50\.0KB of line 1 \(line is 58\.6KB\)\. Full output:/)
|
||||
})
|
||||
|
||||
it('should preserve truncated output when a command times out', async () => {
|
||||
vi.useFakeTimers()
|
||||
const tool = createSandboxedBashTool(mockCwd, mockSessionManager.getSkillEnvs())
|
||||
const mockStdout = new EventEmitter()
|
||||
const mockChild = Object.assign(new EventEmitter(), {
|
||||
stdout: mockStdout,
|
||||
stderr: new EventEmitter(),
|
||||
pid: 103,
|
||||
kill: vi.fn(),
|
||||
})
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- spawn returns mocked ChildProcess
|
||||
;(spawn as any).mockReturnValue(mockChild)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- process.kill return type boolean
|
||||
const killSpy = vi.spyOn(process, 'kill').mockImplementation(() => true as any)
|
||||
|
||||
const signal = createMockSignal()
|
||||
const executePromise = tool.execute('1', { command: 'slow', timeout: 1 }, signal)
|
||||
|
||||
await vi.waitFor(() => expect(spawn).toHaveBeenCalled())
|
||||
|
||||
mockStdout.emit('data', Buffer.from('output before timeout\n'))
|
||||
await vi.advanceTimersByTimeAsync(1100)
|
||||
mockChild.emit('close', null)
|
||||
|
||||
await expect(executePromise).rejects.toThrow(/output before timeout/)
|
||||
await expect(executePromise).rejects.toThrow(/timed out after 1 seconds/)
|
||||
|
||||
vi.useRealTimers()
|
||||
killSpy.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,16 +3,23 @@ import { SandboxManager } from '@anthropic-ai/sandbox-runtime'
|
||||
import { type AgentTool, type AgentToolResult } from '@earendil-works/pi-agent-core'
|
||||
import { Type } from '@sinclair/typebox'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { OutputAccumulator } from '../utils/output-accumulator'
|
||||
import {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
formatSize,
|
||||
type TruncationResult,
|
||||
} from '../utils/truncate'
|
||||
|
||||
const BashParameters = Type.Object({
|
||||
command: Type.String({ description: 'The bash command to execute.' }),
|
||||
timeout: Type.Optional(Type.Number({ description: 'Maximum execution time in seconds.' })),
|
||||
})
|
||||
|
||||
type BashDetails = {
|
||||
export interface BashDetails {
|
||||
exitCode: number | null
|
||||
stdout: string
|
||||
stderr: string
|
||||
truncation?: TruncationResult
|
||||
fullOutputPath?: string
|
||||
}
|
||||
|
||||
export interface SandboxedBashOptions {
|
||||
@@ -20,6 +27,8 @@ export interface SandboxedBashOptions {
|
||||
clearBlockedHost?: () => void
|
||||
}
|
||||
|
||||
const BASH_UPDATE_THROTTLE_MS = 100
|
||||
|
||||
export const createSandboxedBashTool = (
|
||||
cwd: string,
|
||||
skillEnvs: Record<string, string> = {},
|
||||
@@ -27,17 +36,106 @@ export const createSandboxedBashTool = (
|
||||
): AgentTool<typeof BashParameters, BashDetails> => {
|
||||
return {
|
||||
name: 'bash',
|
||||
description: 'Execute a bash command in a sandboxed environment.',
|
||||
description: `Execute a bash command in a sandboxed environment. Returns stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
|
||||
label: 'Executing bash command',
|
||||
parameters: BashParameters,
|
||||
execute: async (
|
||||
toolCallId,
|
||||
{ command, timeout },
|
||||
signal,
|
||||
onUpdate,
|
||||
): Promise<AgentToolResult<BashDetails>> => {
|
||||
options?.clearBlockedHost?.()
|
||||
const wrappedCommand = await SandboxManager.wrapWithSandbox(command)
|
||||
|
||||
const output = new OutputAccumulator({ tempFilePrefix: 'shumai-bash' })
|
||||
let acceptingOutput = true
|
||||
let updateTimer: NodeJS.Timeout | undefined
|
||||
let updateDirty = false
|
||||
let lastUpdateAt = 0
|
||||
|
||||
const emitOutputUpdate = () => {
|
||||
if (!onUpdate || !updateDirty) return
|
||||
updateDirty = false
|
||||
lastUpdateAt = Date.now()
|
||||
const snapshot = output.snapshot({ persistIfTruncated: true })
|
||||
onUpdate({
|
||||
content: [{ type: 'text', text: snapshot.content || '' }],
|
||||
details: {
|
||||
exitCode: null,
|
||||
truncation: snapshot.truncation.truncated ? snapshot.truncation : undefined,
|
||||
fullOutputPath: snapshot.fullOutputPath,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const clearUpdateTimer = () => {
|
||||
if (updateTimer) {
|
||||
clearTimeout(updateTimer)
|
||||
updateTimer = undefined
|
||||
}
|
||||
}
|
||||
|
||||
const scheduleOutputUpdate = () => {
|
||||
if (!onUpdate) return
|
||||
updateDirty = true
|
||||
const delay = BASH_UPDATE_THROTTLE_MS - (Date.now() - lastUpdateAt)
|
||||
if (delay <= 0) {
|
||||
clearUpdateTimer()
|
||||
emitOutputUpdate()
|
||||
return
|
||||
}
|
||||
updateTimer ??= setTimeout(() => {
|
||||
updateTimer = undefined
|
||||
emitOutputUpdate()
|
||||
}, delay)
|
||||
}
|
||||
|
||||
if (onUpdate) {
|
||||
onUpdate({ content: [], details: { exitCode: null } })
|
||||
}
|
||||
|
||||
const handleData = (data: Buffer) => {
|
||||
if (!acceptingOutput) return
|
||||
output.append(data)
|
||||
scheduleOutputUpdate()
|
||||
}
|
||||
|
||||
const finishOutput = async () => {
|
||||
acceptingOutput = false
|
||||
output.finish()
|
||||
clearUpdateTimer()
|
||||
emitOutputUpdate()
|
||||
const snapshot = output.snapshot({ persistIfTruncated: true })
|
||||
await output.closeTempFile()
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const formatOutput = (
|
||||
snapshot: Awaited<ReturnType<typeof finishOutput>>,
|
||||
emptyText = '(no output)',
|
||||
) => {
|
||||
const truncation = snapshot.truncation
|
||||
let text = snapshot.content || emptyText
|
||||
let details: BashDetails | undefined
|
||||
if (truncation.truncated) {
|
||||
details = { exitCode: null, truncation, fullOutputPath: snapshot.fullOutputPath }
|
||||
const startLine = truncation.totalLines - truncation.outputLines + 1
|
||||
const endLine = truncation.totalLines
|
||||
if (truncation.lastLinePartial) {
|
||||
const lastLineSize = formatSize(output.getLastLineBytes())
|
||||
text += `\n\n[Showing last ${formatSize(truncation.outputBytes)} of line ${endLine} (line is ${lastLineSize}). Full output: ${snapshot.fullOutputPath}]`
|
||||
} else if (truncation.truncatedBy === 'lines') {
|
||||
text += `\n\n[Showing lines ${startLine}-${endLine} of ${truncation.totalLines}. Full output: ${snapshot.fullOutputPath}]`
|
||||
} else {
|
||||
text += `\n\n[Showing lines ${startLine}-${endLine} of ${truncation.totalLines} (${formatSize(DEFAULT_MAX_BYTES)} limit). Full output: ${snapshot.fullOutputPath}]`
|
||||
}
|
||||
}
|
||||
return { text, details }
|
||||
}
|
||||
|
||||
const appendStatus = (text: string, status: string) => `${text ? `${text}\n\n` : ''}${status}`
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('bash', ['-c', wrappedCommand], {
|
||||
cwd,
|
||||
@@ -50,8 +148,6 @@ export const createSandboxedBashTool = (
|
||||
},
|
||||
})
|
||||
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let timedOut = false
|
||||
let timeoutHandle: NodeJS.Timeout | undefined
|
||||
|
||||
@@ -68,12 +164,8 @@ export const createSandboxedBashTool = (
|
||||
}, timeout * 1000)
|
||||
}
|
||||
|
||||
child.stdout?.on('data', (data) => {
|
||||
stdout += data.toString()
|
||||
})
|
||||
child.stderr?.on('data', (data) => {
|
||||
stderr += data.toString()
|
||||
})
|
||||
child.stdout?.on('data', handleData)
|
||||
child.stderr?.on('data', handleData)
|
||||
|
||||
const onAbort = () => {
|
||||
if (child.pid) {
|
||||
@@ -87,28 +179,38 @@ export const createSandboxedBashTool = (
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
|
||||
child.on('error', (err) => {
|
||||
child.on('error', async (err) => {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle)
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
const snapshot = await finishOutput()
|
||||
logger.error(
|
||||
{ err, command, stdout, stderr },
|
||||
{ err, command, content: snapshot.content },
|
||||
'Sandboxed bash command encountered process error',
|
||||
)
|
||||
reject(err)
|
||||
})
|
||||
|
||||
child.on('close', (code) => {
|
||||
child.on('close', async (code) => {
|
||||
if (timeoutHandle) clearTimeout(timeoutHandle)
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
|
||||
const blocked = options?.getBlockedHost?.()
|
||||
const snapshot = await finishOutput()
|
||||
const { text: outputText, details: truncationDetails } = formatOutput(snapshot)
|
||||
|
||||
if (signal?.aborted) {
|
||||
logger.error({ command, stdout, stderr }, 'Sandboxed bash command aborted')
|
||||
reject(new Error('aborted'))
|
||||
logger.error({ command, outputText }, 'Sandboxed bash command aborted')
|
||||
reject(new Error(appendStatus(outputText, 'Command aborted')))
|
||||
} else if (timedOut) {
|
||||
logger.error({ timeout, command, stdout, stderr }, 'Sandboxed bash command timed out')
|
||||
reject(new Error(`bash command timed out after ${timeout} seconds`))
|
||||
logger.error({ timeout, command, outputText }, 'Sandboxed bash command timed out')
|
||||
reject(
|
||||
new Error(
|
||||
appendStatus(
|
||||
outputText,
|
||||
`Sandboxed bash command timed out after ${timeout} seconds`,
|
||||
),
|
||||
),
|
||||
)
|
||||
} else if (blocked) {
|
||||
reject(
|
||||
new Error(
|
||||
@@ -117,23 +219,25 @@ export const createSandboxedBashTool = (
|
||||
)
|
||||
} else if (code !== 0 && code !== null) {
|
||||
logger.error(
|
||||
{ exitCode: code, command, stdout, stderr },
|
||||
{ exitCode: code, command, outputText },
|
||||
'Sandboxed bash command exited with failure code',
|
||||
)
|
||||
reject(new Error(`bash command exited with code ${code}`))
|
||||
reject(
|
||||
new Error(
|
||||
appendStatus(outputText, `Sandboxed bash command exited with code ${code}`),
|
||||
),
|
||||
)
|
||||
} else {
|
||||
const output = stdout + stderr
|
||||
resolve({
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: output || (code === 0 ? 'Success' : `Failed with exit code ${code}`),
|
||||
text: outputText,
|
||||
},
|
||||
],
|
||||
details: {
|
||||
exitCode: code,
|
||||
stdout,
|
||||
stderr,
|
||||
...(truncationDetails ?? {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { createWriteStream, type WriteStream } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
DEFAULT_MAX_BYTES,
|
||||
DEFAULT_MAX_LINES,
|
||||
type TruncationResult,
|
||||
truncateTail,
|
||||
} from './truncate'
|
||||
|
||||
export interface OutputAccumulatorOptions {
|
||||
maxLines?: number
|
||||
maxBytes?: number
|
||||
tempFilePrefix?: string
|
||||
}
|
||||
|
||||
export interface OutputSnapshot {
|
||||
content: string
|
||||
truncation: TruncationResult
|
||||
fullOutputPath?: string
|
||||
}
|
||||
|
||||
function defaultTempFilePath(prefix: string): string {
|
||||
const id = randomBytes(8).toString('hex')
|
||||
return join(tmpdir(), `${prefix}-${id}.log`)
|
||||
}
|
||||
|
||||
function byteLength(text: string): number {
|
||||
return Buffer.byteLength(text, 'utf-8')
|
||||
}
|
||||
|
||||
export class OutputAccumulator {
|
||||
private readonly maxLines: number
|
||||
private readonly maxBytes: number
|
||||
private readonly maxRollingBytes: number
|
||||
private readonly tempFilePrefix: string
|
||||
private readonly decoder = new TextDecoder()
|
||||
|
||||
private rawChunks: Buffer[] = []
|
||||
private tailText = ''
|
||||
private tailBytes = 0
|
||||
private tailStartsAtLineBoundary = true
|
||||
private totalRawBytes = 0
|
||||
private totalDecodedBytes = 0
|
||||
private completedLines = 0
|
||||
private totalLines = 0
|
||||
private currentLineBytes = 0
|
||||
private hasOpenLine = false
|
||||
private finished = false
|
||||
|
||||
private tempFilePath: string | undefined
|
||||
private tempFileStream: WriteStream | undefined
|
||||
|
||||
constructor(options: OutputAccumulatorOptions = {}) {
|
||||
this.maxLines = options.maxLines ?? DEFAULT_MAX_LINES
|
||||
this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
|
||||
this.maxRollingBytes = Math.max(this.maxBytes * 2, 1)
|
||||
this.tempFilePrefix = options.tempFilePrefix ?? 'shumai-bash'
|
||||
}
|
||||
|
||||
append(data: Buffer): void {
|
||||
if (this.finished) {
|
||||
throw new Error('Cannot append to a finished output accumulator')
|
||||
}
|
||||
|
||||
this.totalRawBytes += data.length
|
||||
this.appendDecodedText(this.decoder.decode(data, { stream: true }))
|
||||
|
||||
if (this.tempFileStream || this.shouldUseTempFile()) {
|
||||
this.ensureTempFile()
|
||||
this.tempFileStream?.write(data)
|
||||
} else if (data.length > 0) {
|
||||
this.rawChunks.push(data)
|
||||
}
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (this.finished) {
|
||||
return
|
||||
}
|
||||
this.finished = true
|
||||
this.appendDecodedText(this.decoder.decode())
|
||||
if (this.shouldUseTempFile()) {
|
||||
this.ensureTempFile()
|
||||
}
|
||||
}
|
||||
|
||||
snapshot(options: { persistIfTruncated?: boolean } = {}): OutputSnapshot {
|
||||
const tailTruncation = truncateTail(this.getSnapshotText(), {
|
||||
maxLines: this.maxLines,
|
||||
maxBytes: this.maxBytes,
|
||||
})
|
||||
const truncated = this.totalLines > this.maxLines || this.totalDecodedBytes > this.maxBytes
|
||||
const truncatedBy = truncated
|
||||
? (tailTruncation.truncatedBy ?? (this.totalDecodedBytes > this.maxBytes ? 'bytes' : 'lines'))
|
||||
: null
|
||||
const truncation: TruncationResult = {
|
||||
...tailTruncation,
|
||||
truncated,
|
||||
truncatedBy,
|
||||
totalLines: this.totalLines,
|
||||
totalBytes: this.totalDecodedBytes,
|
||||
maxLines: this.maxLines,
|
||||
maxBytes: this.maxBytes,
|
||||
}
|
||||
|
||||
if (options.persistIfTruncated && truncation.truncated) {
|
||||
this.ensureTempFile()
|
||||
}
|
||||
|
||||
return {
|
||||
content: truncation.content,
|
||||
truncation,
|
||||
fullOutputPath: this.tempFilePath,
|
||||
}
|
||||
}
|
||||
|
||||
async closeTempFile(): Promise<void> {
|
||||
if (!this.tempFileStream) {
|
||||
return
|
||||
}
|
||||
|
||||
const stream = this.tempFileStream
|
||||
this.tempFileStream = undefined
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onError = (error: Error) => {
|
||||
stream.off('finish', onFinish)
|
||||
reject(error)
|
||||
}
|
||||
const onFinish = () => {
|
||||
stream.off('error', onError)
|
||||
resolve()
|
||||
}
|
||||
stream.once('error', onError)
|
||||
stream.once('finish', onFinish)
|
||||
stream.end()
|
||||
})
|
||||
}
|
||||
|
||||
getLastLineBytes(): number {
|
||||
return this.currentLineBytes
|
||||
}
|
||||
|
||||
private appendDecodedText(text: string): void {
|
||||
if (text.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const bytes = byteLength(text)
|
||||
this.totalDecodedBytes += bytes
|
||||
this.tailText += text
|
||||
this.tailBytes += bytes
|
||||
if (this.tailBytes > this.maxRollingBytes * 2) {
|
||||
this.trimTail()
|
||||
}
|
||||
|
||||
let newlines = 0
|
||||
let lastNewline = -1
|
||||
for (let i = text.indexOf('\n'); i !== -1; i = text.indexOf('\n', i + 1)) {
|
||||
newlines++
|
||||
lastNewline = i
|
||||
}
|
||||
if (newlines === 0) {
|
||||
this.currentLineBytes += bytes
|
||||
this.hasOpenLine = true
|
||||
} else {
|
||||
this.completedLines += newlines
|
||||
const tail = text.slice(lastNewline + 1)
|
||||
this.currentLineBytes = byteLength(tail)
|
||||
this.hasOpenLine = tail.length > 0
|
||||
}
|
||||
this.totalLines = this.completedLines + (this.hasOpenLine ? 1 : 0)
|
||||
}
|
||||
|
||||
private trimTail(): void {
|
||||
const buffer = Buffer.from(this.tailText, 'utf-8')
|
||||
if (buffer.length <= this.maxRollingBytes) {
|
||||
this.tailBytes = buffer.length
|
||||
return
|
||||
}
|
||||
|
||||
let start = buffer.length - this.maxRollingBytes
|
||||
while (start < buffer.length && (buffer[start] & 0xc0) === 0x80) {
|
||||
start++
|
||||
}
|
||||
|
||||
this.tailStartsAtLineBoundary =
|
||||
start === 0 ? this.tailStartsAtLineBoundary : buffer[start - 1] === 0x0a
|
||||
this.tailText = buffer.subarray(start).toString('utf-8')
|
||||
this.tailBytes = byteLength(this.tailText)
|
||||
}
|
||||
|
||||
private getSnapshotText(): string {
|
||||
if (this.tailStartsAtLineBoundary) {
|
||||
return this.tailText
|
||||
}
|
||||
|
||||
const firstNewline = this.tailText.indexOf('\n')
|
||||
return firstNewline === -1 ? this.tailText : this.tailText.slice(firstNewline + 1)
|
||||
}
|
||||
|
||||
private shouldUseTempFile(): boolean {
|
||||
return (
|
||||
this.totalRawBytes > this.maxBytes ||
|
||||
this.totalDecodedBytes > this.maxBytes ||
|
||||
this.totalLines > this.maxLines
|
||||
)
|
||||
}
|
||||
|
||||
private ensureTempFile(): void {
|
||||
if (this.tempFilePath) {
|
||||
return
|
||||
}
|
||||
this.tempFilePath = defaultTempFilePath(this.tempFilePrefix)
|
||||
this.tempFileStream = createWriteStream(this.tempFilePath)
|
||||
for (const chunk of this.rawChunks) {
|
||||
this.tempFileStream.write(chunk)
|
||||
}
|
||||
this.rawChunks = []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
export const DEFAULT_MAX_LINES = 2000
|
||||
export const DEFAULT_MAX_BYTES = 50 * 1024 // 50KB
|
||||
|
||||
export interface TruncationResult {
|
||||
content: string
|
||||
truncated: boolean
|
||||
truncatedBy: 'lines' | 'bytes' | null
|
||||
totalLines: number
|
||||
totalBytes: number
|
||||
outputLines: number
|
||||
outputBytes: number
|
||||
lastLinePartial: boolean
|
||||
firstLineExceedsLimit: boolean
|
||||
maxLines: number
|
||||
maxBytes: number
|
||||
}
|
||||
|
||||
export interface TruncationOptions {
|
||||
maxLines?: number
|
||||
maxBytes?: number
|
||||
}
|
||||
|
||||
export function formatSize(bytes: number): string {
|
||||
if (bytes < 1024) {
|
||||
return `${bytes}B`
|
||||
} else if (bytes < 1024 * 1024) {
|
||||
return `${(bytes / 1024).toFixed(1)}KB`
|
||||
} else {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`
|
||||
}
|
||||
}
|
||||
|
||||
function splitLinesForCounting(content: string): string[] {
|
||||
if (content.length === 0) {
|
||||
return []
|
||||
}
|
||||
const lines = content.split('\n')
|
||||
if (content.endsWith('\n')) {
|
||||
lines.pop()
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function truncateStringToBytesFromEnd(str: string, maxBytes: number): string {
|
||||
const buf = Buffer.from(str, 'utf-8')
|
||||
if (buf.length <= maxBytes) {
|
||||
return str
|
||||
}
|
||||
|
||||
let start = buf.length - maxBytes
|
||||
while (start < buf.length && (buf[start] & 0xc0) === 0x80) {
|
||||
start++
|
||||
}
|
||||
|
||||
return buf.subarray(start).toString('utf-8')
|
||||
}
|
||||
|
||||
export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult {
|
||||
const maxLines = options.maxLines ?? DEFAULT_MAX_LINES
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
|
||||
|
||||
const totalBytes = Buffer.byteLength(content, 'utf-8')
|
||||
const lines = splitLinesForCounting(content)
|
||||
const totalLines = lines.length
|
||||
|
||||
if (totalLines <= maxLines && totalBytes <= maxBytes) {
|
||||
return {
|
||||
content,
|
||||
truncated: false,
|
||||
truncatedBy: null,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: totalLines,
|
||||
outputBytes: totalBytes,
|
||||
lastLinePartial: false,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
}
|
||||
}
|
||||
|
||||
const outputLinesArr: string[] = []
|
||||
let outputBytesCount = 0
|
||||
let truncatedBy: 'lines' | 'bytes' = 'lines'
|
||||
let lastLinePartial = false
|
||||
|
||||
for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) {
|
||||
const line = lines[i]
|
||||
const lineBytes = Buffer.byteLength(line, 'utf-8') + (outputLinesArr.length > 0 ? 1 : 0)
|
||||
|
||||
if (outputBytesCount + lineBytes > maxBytes) {
|
||||
truncatedBy = 'bytes'
|
||||
if (outputLinesArr.length === 0) {
|
||||
const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes)
|
||||
outputLinesArr.unshift(truncatedLine)
|
||||
outputBytesCount = Buffer.byteLength(truncatedLine, 'utf-8')
|
||||
lastLinePartial = true
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
outputLinesArr.unshift(line)
|
||||
outputBytesCount += lineBytes
|
||||
}
|
||||
|
||||
if (outputLinesArr.length >= maxLines && outputBytesCount <= maxBytes) {
|
||||
truncatedBy = 'lines'
|
||||
}
|
||||
|
||||
const outputContent = outputLinesArr.join('\n')
|
||||
const finalOutputBytes = Buffer.byteLength(outputContent, 'utf-8')
|
||||
|
||||
return {
|
||||
content: outputContent,
|
||||
truncated: true,
|
||||
truncatedBy,
|
||||
totalLines,
|
||||
totalBytes,
|
||||
outputLines: outputLinesArr.length,
|
||||
outputBytes: finalOutputBytes,
|
||||
lastLinePartial,
|
||||
firstLineExceedsLimit: false,
|
||||
maxLines,
|
||||
maxBytes,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user