mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 17:48:25 +00:00
feat(agent): strip and restore skill content and preserve envs on session rebuild
This commit is contained in:
@@ -4,6 +4,9 @@ import { s3Service } from '@/services/s3/s3'
|
||||
import { type SessionTreeEntry } from '@earendil-works/pi-agent-core'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { agentService } from '@/services/agent/agent'
|
||||
import { createAgentSession } from './index'
|
||||
import * as sandboxedBashModule from './tools/sandboxed-bash'
|
||||
|
||||
describe('DatabaseSessionStorage', () => {
|
||||
setupTestDbHooks()
|
||||
@@ -230,6 +233,108 @@ describe('DatabaseSessionStorage', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('should strip and reinject skill content', async () => {
|
||||
const { agent } = await setupTestData()
|
||||
const storage = await DatabaseSessionStorage.create({ agentId: agent.id })
|
||||
|
||||
const getSkillContentSpy = vi
|
||||
.spyOn(agentService, 'getSkillContent')
|
||||
.mockResolvedValue('# Mocked Skill Content')
|
||||
|
||||
const entry: SessionTreeEntry = {
|
||||
type: 'message',
|
||||
id: 'msg-skill',
|
||||
parentId: null,
|
||||
timestamp: new Date().toISOString(),
|
||||
message: {
|
||||
role: 'toolResult',
|
||||
toolCallId: 'call-2',
|
||||
toolName: 'read_skill',
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
content: [{ type: 'text', text: 'original-skill-content' }],
|
||||
details: {
|
||||
skillId: 'some-skill-id',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
await storage.appendEntry(entry)
|
||||
|
||||
// Verify saved entry is stripped
|
||||
const record = await prisma.agentSessionEntry.findUnique({ where: { id: 'msg-skill' } })
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const savedEntry = record?.entry as any
|
||||
expect(savedEntry.message.content[0].text).toBe('__SKILL_CONTENT__')
|
||||
|
||||
// Verify retrieval reinjects data
|
||||
const retrieved = await storage.getEntry('msg-skill')
|
||||
expect(getSkillContentSpy).toHaveBeenCalledWith('some-skill-id')
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect((retrieved as any).message.content[0].text).toBe('# Mocked Skill Content')
|
||||
})
|
||||
|
||||
it('should restore environment variables when resuming session', async () => {
|
||||
const { agent, user } = await setupTestData()
|
||||
|
||||
// 1. Create a session first to get a sessionId
|
||||
const initialStorage = await DatabaseSessionStorage.create({ agentId: agent.id })
|
||||
const sessionId = initialStorage.sessionId
|
||||
|
||||
// 2. Append a successful read_skill entry
|
||||
const entry: SessionTreeEntry = {
|
||||
type: 'message',
|
||||
id: 'msg-skill-env',
|
||||
parentId: null,
|
||||
timestamp: new Date().toISOString(),
|
||||
message: {
|
||||
role: 'toolResult',
|
||||
toolCallId: 'call-env-1',
|
||||
toolName: 'read_skill',
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
content: [{ type: 'text', text: 'skill content' }],
|
||||
details: {
|
||||
skillId: 'env-skill-id',
|
||||
},
|
||||
},
|
||||
}
|
||||
await initialStorage.appendEntry(entry)
|
||||
|
||||
// 3. Mock getSkillEnvs to return environment variables
|
||||
const getSkillEnvsSpy = vi.spyOn(agentService, 'getSkillEnvs').mockResolvedValue({
|
||||
MOCK_ENV_VAR: 'mocked-value',
|
||||
})
|
||||
|
||||
// 4. Mock agentService.getSkillContent to avoid actual download throws
|
||||
vi.spyOn(agentService, 'getSkillContent').mockResolvedValue('# Skill')
|
||||
|
||||
// 5. Spy on createSandboxedBashTool
|
||||
const createSandboxedBashToolSpy = vi.spyOn(sandboxedBashModule, 'createSandboxedBashTool')
|
||||
|
||||
// 6. Resume the session via createAgentSession
|
||||
await createAgentSession({
|
||||
agentId: agent.id,
|
||||
providerName: 'test-provider',
|
||||
modelId: 'test-model',
|
||||
systemPrompt: 'Test prompt',
|
||||
teamSkills: [],
|
||||
allowedDomains: [],
|
||||
sessionId: sessionId,
|
||||
userId: user.id,
|
||||
providers: [],
|
||||
})
|
||||
|
||||
// 7. Verify skill environment variables are restored.
|
||||
expect(getSkillEnvsSpy).toHaveBeenCalledWith('env-skill-id')
|
||||
expect(createSandboxedBashToolSpy).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
MOCK_ENV_VAR: 'mocked-value',
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('should retrieve label', async () => {
|
||||
const { agent } = await setupTestData()
|
||||
const storage = await DatabaseSessionStorage.create({ agentId: agent.id })
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
type SessionTreeEntry,
|
||||
type SessionTreeEntryBase,
|
||||
} from '@earendil-works/pi-agent-core'
|
||||
import { agentService } from '@/services/agent/agent'
|
||||
|
||||
export interface DatabaseSessionMetadata extends SessionMetadata {
|
||||
agentId: string
|
||||
@@ -54,11 +55,13 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
async appendEntry(entry: SessionTreeEntry): Promise<void> {
|
||||
const strippedEntry = structuredClone(entry)
|
||||
|
||||
// Strip image data before saving to DB
|
||||
// Strip image data and skill content before saving to DB
|
||||
if (strippedEntry.type === 'message') {
|
||||
const msg = strippedEntry.message
|
||||
if (msg.role === 'toolResult') {
|
||||
const details = msg.details as { sourceKeys?: string[] } | undefined
|
||||
const details = msg.details as { sourceKeys?: string[]; skillId?: string } | undefined
|
||||
|
||||
// Strip S3 image data
|
||||
if (Array.isArray(msg.content) && details?.sourceKeys) {
|
||||
const sourceKeys = details.sourceKeys
|
||||
let keyIdx = 0
|
||||
@@ -69,6 +72,23 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Strip skill content
|
||||
if (msg.toolName === 'read_skill' && !msg.isError) {
|
||||
if (details?.skillId && Array.isArray(msg.content)) {
|
||||
msg.content.forEach((item: { type: string; text?: string }) => {
|
||||
if (
|
||||
item.type === 'text' &&
|
||||
item.text &&
|
||||
!item.text.startsWith('Skill with ID') &&
|
||||
!item.text.startsWith('Skill downloaded but') &&
|
||||
!item.text.startsWith('Error reading skill')
|
||||
) {
|
||||
item.text = '__SKILL_CONTENT__'
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,6 +111,7 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
if (!record) return undefined
|
||||
const entry = record.entry as unknown as SessionTreeEntry
|
||||
await this.reinjectImageDataAsync(entry)
|
||||
await this.reinjectSkillContentAsync(entry)
|
||||
return entry
|
||||
}
|
||||
|
||||
@@ -112,6 +133,7 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
>
|
||||
for (const entry of entries) {
|
||||
await this.reinjectImageDataAsync(entry)
|
||||
await this.reinjectSkillContentAsync(entry)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
@@ -158,6 +180,7 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
// Only reinject image data (which may involve S3 calls) for the entries in the path
|
||||
for (const entry of pathEntries) {
|
||||
await this.reinjectImageDataAsync(entry)
|
||||
await this.reinjectSkillContentAsync(entry)
|
||||
}
|
||||
|
||||
return pathEntries
|
||||
@@ -171,6 +194,7 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
const entries = records.map((r) => r.entry as unknown as SessionTreeEntry)
|
||||
for (const entry of entries) {
|
||||
await this.reinjectImageDataAsync(entry)
|
||||
await this.reinjectSkillContentAsync(entry)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
@@ -200,6 +224,30 @@ export class DatabaseSessionStorage implements SessionStorage<DatabaseSessionMet
|
||||
}
|
||||
}
|
||||
|
||||
private async reinjectSkillContentAsync(entry: SessionTreeEntry) {
|
||||
if (entry.type === 'message') {
|
||||
const msg = entry.message as AgentMessage
|
||||
if (msg.role === 'toolResult' && msg.toolName === 'read_skill') {
|
||||
const details = msg.details as { skillId?: string } | undefined
|
||||
if (details?.skillId && Array.isArray(msg.content)) {
|
||||
for (const item of msg.content) {
|
||||
if (item.type === 'text' && item.text === '__SKILL_CONTENT__') {
|
||||
try {
|
||||
const content = await agentService.getSkillContent(details.skillId)
|
||||
item.text = content
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`Failed to re-inject skill content for skill ${details.skillId}:`,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static async create(params: {
|
||||
agentId: string
|
||||
userId?: string
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getModel } from '@earendil-works/pi-ai'
|
||||
import { Type, type TSchema } from '@sinclair/typebox'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { agentService } from '@/services/agent/agent'
|
||||
import { DatabaseSessionStorage } from './database-session-storage'
|
||||
import { analyzeAssetMediaTool } from './tools/analyze-asset-media'
|
||||
import { createCreateFileTool } from './tools/create-file'
|
||||
@@ -129,6 +130,29 @@ export async function createAgentSession(params: CreateAgentSessionParams) {
|
||||
Object.assign(skillEnvs, envs)
|
||||
}
|
||||
|
||||
// Restore env variables from previously loaded skills in this session
|
||||
if (sessionId) {
|
||||
try {
|
||||
const entries = await storage.getEntries()
|
||||
for (const entry of entries) {
|
||||
if (
|
||||
entry.type === 'message' &&
|
||||
entry.message.role === 'toolResult' &&
|
||||
entry.message.toolName === 'read_skill' &&
|
||||
!entry.message.isError
|
||||
) {
|
||||
const details = entry.message.details as { skillId?: string } | undefined
|
||||
if (details?.skillId) {
|
||||
const envs = await agentService.getSkillEnvs(details.skillId)
|
||||
onEnvsAdded(envs)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to restore skill environment variables:', err)
|
||||
}
|
||||
}
|
||||
|
||||
const sandboxedBash = createSandboxedBashTool(process.cwd(), skillEnvs)
|
||||
const readSkill = createReadSkillTool(onEnvsAdded)
|
||||
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
import { prisma } from '@/db'
|
||||
import { s3Service } from '@/services/s3/s3'
|
||||
import { Type } from '@sinclair/typebox'
|
||||
import { type AgentTool } from '@earendil-works/pi-agent-core'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
import AdmZip from 'adm-zip'
|
||||
import { agentService } from '@/services/agent/agent'
|
||||
|
||||
const readSkillSchema = Type.Object({
|
||||
skillId: Type.String({ description: 'The ID of the skill to read' }),
|
||||
@@ -30,78 +27,13 @@ export const createReadSkillTool = (
|
||||
}
|
||||
}
|
||||
|
||||
// Capture environment variables if present in config
|
||||
const config = skill.config as unknown as PrismaJson.SkillConfig
|
||||
if (config?.environmentVariables && Array.isArray(config.environmentVariables)) {
|
||||
const envs: Record<string, string> = {}
|
||||
for (const envVar of config.environmentVariables) {
|
||||
const value =
|
||||
envVar.default !== undefined && envVar.default !== null && envVar.default !== ''
|
||||
? envVar.default
|
||||
: process.env[envVar.name]
|
||||
if (value !== undefined) {
|
||||
envs[envVar.name] = value
|
||||
}
|
||||
}
|
||||
onEnvsAdded(envs)
|
||||
}
|
||||
// Capture environment variables using agentService
|
||||
const envs = await agentService.getSkillEnvs(params.skillId)
|
||||
onEnvsAdded(envs)
|
||||
|
||||
const skillDir = path.join(process.cwd(), '.pi', 'skills', skill.id)
|
||||
const hashFile = path.join(skillDir, '.hash')
|
||||
let needsDownload = true
|
||||
// Retrieve skill content using agentService
|
||||
const skillMdContent = await agentService.getSkillContent(params.skillId)
|
||||
|
||||
if (fs.existsSync(hashFile)) {
|
||||
const localHash = fs.readFileSync(hashFile, 'utf8')
|
||||
if (localHash === skill.hash) {
|
||||
needsDownload = false
|
||||
}
|
||||
}
|
||||
|
||||
if (needsDownload) {
|
||||
if (fs.existsSync(skillDir)) {
|
||||
fs.rmSync(skillDir, { recursive: true, force: true })
|
||||
}
|
||||
fs.mkdirSync(skillDir, { recursive: true })
|
||||
|
||||
const asset = await prisma.asset.findUnique({
|
||||
where: { id: skill.assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset || !asset.storageKey?.key) {
|
||||
throw new Error('Skill asset not found or has no key')
|
||||
}
|
||||
|
||||
const { buffer: zipBuffer } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
asset.storageKey.key,
|
||||
)
|
||||
|
||||
const zip = new AdmZip(zipBuffer)
|
||||
zip.extractAllTo(skillDir, true)
|
||||
|
||||
fs.writeFileSync(hashFile, skill.hash)
|
||||
}
|
||||
|
||||
// Read SKILL.md
|
||||
let skillMdPath = path.join(skillDir, 'SKILL.md')
|
||||
if (!fs.existsSync(skillMdPath)) {
|
||||
skillMdPath = path.join(skillDir, 'skill.md')
|
||||
}
|
||||
|
||||
if (!fs.existsSync(skillMdPath)) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Skill downloaded but SKILL.md not found in ${skillDir}`,
|
||||
},
|
||||
],
|
||||
details: { skillId: params.skillId },
|
||||
}
|
||||
}
|
||||
|
||||
const skillMdContent = fs.readFileSync(skillMdPath, 'utf8')
|
||||
return {
|
||||
content: [{ type: 'text', text: skillMdContent }],
|
||||
details: { skillId: params.skillId },
|
||||
|
||||
@@ -6,10 +6,95 @@ import {
|
||||
UpdateAgentParams,
|
||||
} from '@/dtos/agent'
|
||||
import '@/prisma-json-types'
|
||||
import { s3Service } from '@/services/s3/s3'
|
||||
import AdmZip from 'adm-zip'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
|
||||
export class AgentService {
|
||||
constructor(private readonly prismaClient: typeof prisma = prisma) {}
|
||||
|
||||
async getSkillContent(skillId: string): Promise<string> {
|
||||
const skill = await this.prismaClient.skill.findUnique({
|
||||
where: { id: skillId },
|
||||
})
|
||||
|
||||
if (!skill) {
|
||||
throw new Error(`Skill with ID ${skillId} not found.`)
|
||||
}
|
||||
|
||||
const skillDir = path.join(process.cwd(), '.pi', 'skills', skill.id)
|
||||
const hashFile = path.join(skillDir, '.hash')
|
||||
let needsDownload = true
|
||||
|
||||
if (fs.existsSync(hashFile)) {
|
||||
const localHash = fs.readFileSync(hashFile, 'utf8')
|
||||
if (localHash === skill.hash) {
|
||||
needsDownload = false
|
||||
}
|
||||
}
|
||||
|
||||
if (needsDownload) {
|
||||
if (fs.existsSync(skillDir)) {
|
||||
fs.rmSync(skillDir, { recursive: true, force: true })
|
||||
}
|
||||
fs.mkdirSync(skillDir, { recursive: true })
|
||||
|
||||
const asset = await this.prismaClient.asset.findUnique({
|
||||
where: { id: skill.assetId },
|
||||
include: { storageKey: true },
|
||||
})
|
||||
|
||||
if (!asset || !asset.storageKey?.key) {
|
||||
throw new Error('Skill asset not found or has no key')
|
||||
}
|
||||
|
||||
const { buffer: zipBuffer } = await s3Service.getObject(
|
||||
process.env.S3_BUCKET || 'shumai',
|
||||
asset.storageKey.key,
|
||||
)
|
||||
|
||||
const zip = new AdmZip(zipBuffer)
|
||||
zip.extractAllTo(skillDir, true)
|
||||
|
||||
fs.writeFileSync(hashFile, skill.hash)
|
||||
}
|
||||
|
||||
// Read SKILL.md
|
||||
let skillMdPath = path.join(skillDir, 'SKILL.md')
|
||||
if (!fs.existsSync(skillMdPath)) {
|
||||
skillMdPath = path.join(skillDir, 'skill.md')
|
||||
}
|
||||
|
||||
if (!fs.existsSync(skillMdPath)) {
|
||||
throw new Error(`Skill downloaded but SKILL.md not found in ${skillDir}`)
|
||||
}
|
||||
|
||||
return fs.readFileSync(skillMdPath, 'utf8')
|
||||
}
|
||||
|
||||
async getSkillEnvs(skillId: string): Promise<Record<string, string>> {
|
||||
const skill = await this.prismaClient.skill.findUnique({
|
||||
where: { id: skillId },
|
||||
})
|
||||
if (!skill) return {}
|
||||
|
||||
const config = skill.config as unknown as PrismaJson.SkillConfig
|
||||
const envs: Record<string, string> = {}
|
||||
if (config?.environmentVariables && Array.isArray(config.environmentVariables)) {
|
||||
for (const envVar of config.environmentVariables) {
|
||||
const value =
|
||||
envVar.default !== undefined && envVar.default !== null && envVar.default !== ''
|
||||
? envVar.default
|
||||
: process.env[envVar.name]
|
||||
if (value !== undefined) {
|
||||
envs[envVar.name] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
return envs
|
||||
}
|
||||
|
||||
async createAgent(params: CreateAgentParams) {
|
||||
const {
|
||||
teamId,
|
||||
|
||||
Reference in New Issue
Block a user