fix(runner): enforce ownership and safe reconnects

Gate dispatch on reconciliation, retain cancellations across disconnects, and stop replaced clients. Guard the single-server runner topology with a database lease. Restage missing generated inputs, report signal exits, and bound generated-file cleanup.
This commit is contained in:
yiling
2026-11-18 12:04:29 +08:00
parent fd4dc211f8
commit a9281cbc75
23 changed files with 691 additions and 128 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
### Added
- **cli**: Run remote shell commands and exchange files with `shumai-cli runner start` on macOS, Linux, and Windows, with optional SRT sandboxing. On Windows, the runner automatically uses Bash when available and falls back to Command Prompt.
- **agent**: Support binding external Runners per team so agents can execute shell commands and exchange files securely over WebSocket without requiring a built-in server sandbox, and allow assigning a default or dedicated Runner per agent in Team Settings.
- **agent**: Connect remote runners on single-server deployments to run shell commands and exchange files. Assign the team's default runner or a dedicated runner in Agent Settings. A runner's workspace is shared by everyone who can use it.
- **i18n**: Add Korean as a display language, selectable on the sign-up page and in team settings.
### Fixed
+38 -12
View File
@@ -17,17 +17,21 @@ describe('runner WebSocket client', () => {
let messages: RunnerToServerMessage[]
let welcomeInFlight: string[]
let connections: number
let replacePrevious: boolean
let connectRunner: () => RunnerClient
beforeEach(async () => {
cwd = await mkdtemp(path.join(tmpdir(), 'shumai-client-test-'))
messages = []
welcomeInFlight = []
connections = 0
replacePrevious = false
server = new WebSocketServer({ host: '127.0.0.1', port: 0 })
await new Promise<void>((resolve) => server.once('listening', resolve))
const address = server.address()
if (!address || typeof address === 'string') throw new Error('No WebSocket address')
server.on('connection', (connection, request) => {
expect(request.headers.authorization).toBe('Bearer runner-key')
if (replacePrevious) socket.close(4000, 'Replaced by a newer connection')
socket = connection
connections++
connection.on('message', (raw) =>
@@ -43,18 +47,21 @@ describe('runner WebSocket client', () => {
}),
)
})
const files = new RunnerFiles(cwd)
runner = new RunnerClient(
{
url: new URL(`ws://127.0.0.1:${address.port}/api/runners/ws`),
key: 'runner-key',
cwd,
srt: undefined,
},
files,
new RunnerShell(files),
() => {},
)
connectRunner = () => {
const files = new RunnerFiles(cwd)
return new RunnerClient(
{
url: new URL(`ws://127.0.0.1:${address.port}/api/runners/ws`),
key: 'runner-key',
cwd,
srt: undefined,
},
files,
new RunnerShell(files),
() => {},
)
}
runner = connectRunner()
running = runner.run()
// Attach an error handler immediately, including for tests exercising terminal errors.
void running.catch(() => {})
@@ -161,6 +168,25 @@ describe('runner WebSocket client', () => {
expect(connections).toBe(1)
})
it('stops the superseded client when a second runner connects with the same key', async () => {
replacePrevious = true
let replaced: unknown
void running.catch((error: unknown) => {
replaced = error
})
const replacement = connectRunner()
const replacementRun = replacement.run()
void replacementRun.catch(() => {})
try {
await vi.waitFor(() => expect(replaced).toBeInstanceOf(Error))
await expect(running).rejects.toThrow('replaced')
expect(connections).toBe(2)
} finally {
replacement.stop()
await replacementRun.catch(() => {})
}
})
it('rejects invalid messages and path-like execution IDs', async () => {
send({ type: 'exec.start', executionId: '../escape', command: 'echo unsafe' })
await expect(running).rejects.toThrow('invalid runner protocol')
+2 -1
View File
@@ -1,6 +1,7 @@
import WebSocket from 'ws'
import { setTimeout as delay } from 'node:timers/promises'
import {
RUNNER_REPLACED_CLOSE_CODE,
serverToRunnerMessageSchema,
type RunnerExecOutputMessage,
type RunnerExecResultMessage,
@@ -101,7 +102,7 @@ export class RunnerClient {
socket.on('close', (code) => {
clearInterval(heartbeat)
this.ready = false
if (code === 1008 && !this.stopped.signal.aborted)
if ((code === 1008 || code === RUNNER_REPLACED_CLOSE_CODE) && !this.stopped.signal.aborted)
reject(
new CliError('Runner access was revoked or another connection replaced this runner.'),
)
+11
View File
@@ -81,6 +81,16 @@ if (regenCmdIndex !== -1) {
async function run() {
validateArchiveStorageConfig()
const runnerLease = await runnerService.startServerTopology()
runnerLease.addEventListener(
'abort',
() => {
console.error('Lost the runner server lease:', runnerLease.reason)
process.exit(1)
},
{ once: true },
)
runnerLease.throwIfAborted()
// Initialize workflows and activities for local executor mode
initAgentWorkflows()
initTranscodeWorkflows()
@@ -222,6 +232,7 @@ async function run() {
await workflowService.shutdownWorkers().catch(() => {})
assetService.stopCleanupJob()
server.stop(true)
await runnerService.stopServerTopology().catch(console.error)
process.exit(0)
}
+2
View File
@@ -72,6 +72,8 @@ When running multiple replicas of the transcode workers or main servers across m
- **Custom Worker ID**: Set `SHUMAI_WORKERS_ID` (or `SHUMAI_WORKER_ID`) if you wish to assign an explicit identifier.
- **Kubernetes**: In Kubernetes Pods, set `POD_NAME` (via Downward API `fieldRef: { fieldPath: metadata.name }`) to provide stable, unique per-pod identity across pod restarts.
Agent runners require a single main server per database. With runners configured, stop the old main server before starting its replacement. For Kubernetes, use a single main server replica with the Recreate deployment strategy. Transcode workers can still scale independently.
---
## Scalable Deployment on Kubernetes & AWS EKS
+4
View File
@@ -5,6 +5,8 @@ description: Connect an optional command runner and configure sandboxing on its
Agents use Shumai's built-in tools without a runner. To execute shell commands or download assets into a shell workspace, connect a **runner** on a machine you control. The runner opens an authenticated outbound WebSocket connection; no inbound port is needed.
Agent runners require one main Shumai server per database. Multiple main servers can share a database while it has no configured runners. Stop the other main servers before creating a runner. Transcode workers can run separately.
## Connect a runner
1. In **Settings → Agent Runners**, create a runner and copy its one-time key. Only team owners can manage runners and rotate their keys.
@@ -33,6 +35,8 @@ The `bash` tool uses `/bin/sh` on macOS and Linux. On Windows, the runner looks
Commands start in the workspace directory. Skills, downloaded assets, generated files, and output logs live under `.pi`. Commands inherit baseline OS settings and explicit skill variables, rather than the runner's full environment, including its runner key and personal API token.
Everyone with shell access to the same runner can read files left in its workspace, including assets and logs from other projects. Shumai project permissions apply when assets are downloaded. Share a runner only among users who may read all files in that workspace. For separate trust groups, use separate runner accounts or virtual machines and restrict access to the agents bound to each runner.
## Optional SRT sandbox
Without `--srt`, commands have the runner operating-system user's permissions. Connect only to a trusted Shumai server; use a dedicated container, VM, or account when appropriate.
+2
View File
@@ -53,6 +53,8 @@ Worker 根据可配置的并发上限从 PostgreSQL 队列中拉取任务:
- **自定义 Worker ID**:若需显式指定标识符,可设置 `SHUMAI_WORKERS_ID`(或 `SHUMAI_WORKER_ID`)。
- **Kubernetes 环境**:在 Kubernetes Pod 中,可通过 Downward API(`fieldRef: { fieldPath: metadata.name }`)设置 `POD_NAME`,以在 Pod 重启期间保持稳定且唯一的单 Pod 身份标识。
使用智能体运行器时,每个数据库只允许运行一个主服务器。已配置运行器时,请先停止旧主服务器,再启动替换实例。在 Kubernetes 中,主服务器应使用单副本和 Recreate 部署策略。转码 Worker 仍可独立扩展。
---
## 基于 Kubernetes 与 AWS EKS 的弹性扩展部署
+4
View File
@@ -5,6 +5,8 @@ description: 连接可选的命令运行器,并在运行器机器上配置沙
智能体无需运行器即可使用内置工具。如需执行命令或将素材下载到命令工作区,请在自己管理的机器上连接**运行器**。运行器主动通过经过身份验证的 WebSocket 连接 Shumai,无需开放入站端口。
每个数据库只支持一台主 Shumai 服务器使用智能体运行器。未配置运行器时,多台主服务器可以共享数据库。创建运行器前,请停止其他主服务器副本。转码 Worker 可以独立运行。
## 连接运行器
1. 在 **设置 → 智能体运行器** 中创建运行器,复制仅显示一次的密钥。只有团队所有者可以管理运行器和轮换密钥。
@@ -31,6 +33,8 @@ shumai-cli runner start --cwd C:\workspace
命令从工作目录启动,技能、下载素材、生成文件和日志位于 `.pi` 中。子进程只继承基础系统环境及明确配置的技能变量,不自动继承运行器密钥、个人 API 令牌或其他任意环境变量。
所有有权在同一运行器执行命令的用户都可以读取工作区中保留的文件,包括其他项目的素材和日志。Shumai 下载素材时会检查项目权限。请仅在有权读取工作区内所有文件的用户之间共享运行器。不同信任范围应使用独立的运行器账号或虚拟机,并限制对应智能体的访问权限。
## 可选 SRT 沙箱
未使用 `--srt` 时,命令具有运行器系统用户的权限。只连接可信 Shumai 服务器,可按需使用独立容器、虚拟机或账号。
@@ -43,6 +43,19 @@ function createMockWs() {
}
}
function reconcile(runnerId: string, ws: RunnerSocket) {
runnerService.handleMessage(
runnerId,
ws,
JSON.stringify({
type: 'reconcile',
environment: { platform: 'linux', shell: '/bin/sh' },
activeExecutions: [],
completedExecutions: [],
}),
)
}
describe('Concurrent Runner Sessions', () => {
setupTestDbHooks()
@@ -64,6 +77,7 @@ describe('Concurrent Runner Sessions', () => {
{ id: runner.id, teamId: team.id, name: runner.name, keyHash: hashRunnerKey(key) },
conn.ws,
)
reconcile(runner.id, conn.ws)
await prisma.user.create({
data: {
@@ -201,6 +215,8 @@ describe('Concurrent Runner Sessions', () => {
},
gpuConn.ws,
)
reconcile(defaultRunner.id, defaultConn.ws)
reconcile(gpuRunner.id, gpuConn.ws)
const defaultTool = createRunnerBashTool(
{},
@@ -186,6 +186,21 @@ describe('createRunnerBashTool', () => {
).rejects.toThrow(/exited with code 1/)
})
it('reports signal termination as a command failure', async () => {
const tool = createRunnerBashTool({}, { teamId: mockTeamId })
vi.mocked(runnerService.exec).mockResolvedValue({
executionId: 'signaled',
output: '',
exitCode: null,
signal: 'SIGTERM',
timedOut: false,
aborted: false,
})
await expect(invokeTool(tool, '1', { command: 'command' }, createMockSignal())).rejects.toThrow(
'terminated by SIGTERM',
)
})
it('should handle timeouts', async () => {
const tool = createRunnerBashTool({}, { teamId: mockTeamId })
+8
View File
@@ -208,6 +208,14 @@ export const createRunnerBashTool = (
throw new Error(appendStatus(outputText, execResult.error))
}
if (execResult.signal) {
logger.error(
{ signal: execResult.signal, command, outputText },
'Bash command was terminated',
)
throw new Error(appendStatus(outputText, 'Bash command terminated by ' + execResult.signal))
}
if (execResult.exitCode !== 0 && execResult.exitCode !== null) {
logger.error(
{ exitCode: execResult.exitCode, command, outputText },
+27
View File
@@ -8,6 +8,7 @@ import { s3Service, getStorageBackend } from '@shumai/core/src/s3/s3'
import * as archiveStorageModule from '@shumai/core/src/s3/archive-storage'
import { uploadService } from '@shumai/core/src/upload/upload'
import { metadataService } from '@shumai/core/src/metadata/metadata'
import { watermarkService } from '@shumai/core/src/watermark/watermark'
import { AssetService } from './asset'
import { ensureJpegInStorage } from '@shumai/core/src/s3/preview-converter'
@@ -22,6 +23,7 @@ vi.mock('@shumai/core/src/s3/s3', () => ({
deletePrefix: vi.fn().mockResolvedValue(1),
copyObject: vi.fn().mockResolvedValue(undefined),
getStoragePath: vi.fn((bucket: string, key: string) => `/mock/data/${bucket}/${key}`),
listObjects: vi.fn().mockResolvedValue([]),
},
getStorageBackend: vi.fn().mockReturnValue('local'),
}))
@@ -47,6 +49,31 @@ describe('AssetService', () => {
vi.clearAllMocks()
})
it('checks generated file expiration hourly while continuing regular cleanup', async () => {
vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] })
const startedAt = Date.now()
const regularCleanup = vi
.spyOn(watermarkService, 'purgeOrphanWatermarkConfigs')
.mockResolvedValue(0)
try {
await assetService.startCleanupJob()
await vi.waitFor(() => expect(s3Service.listObjects).toHaveBeenCalledTimes(1))
expect(s3Service.listObjects).toHaveBeenCalledWith(expect.any(String), 'temp/generated/')
await vi.advanceTimersByTimeAsync(5_000)
await vi.waitFor(() => expect(regularCleanup).toHaveBeenCalledTimes(2))
expect(s3Service.listObjects).toHaveBeenCalledTimes(1)
vi.setSystemTime(startedAt + 60 * 60 * 1000)
await vi.advanceTimersByTimeAsync(5_000)
await vi.waitFor(() => expect(s3Service.listObjects).toHaveBeenCalledTimes(2))
} finally {
assetService.stopCleanupJob()
regularCleanup.mockRestore()
vi.useRealTimers()
}
})
const setupBasicAssets = async () => {
const user = await prisma.user.create({
data: {
+7 -1
View File
@@ -1585,7 +1585,9 @@ export class AssetService {
private cleanupJobsRunning = false
async startCleanupJob() {
if (this.cleanupJobsRunning) return
this.cleanupJobsRunning = true
let nextGeneratedCleanupAt = 0
const runCleanup = async () => {
if (!this.cleanupJobsRunning) return
@@ -1594,8 +1596,12 @@ export class AssetService {
await this.expireTrashedAssets()
await this.purgePendingAssets()
await this.purgeUnreferencedStorageKeys()
await this.purgeExpiredGeneratedFiles()
await watermarkService.purgeOrphanWatermarkConfigs()
const now = Date.now()
if (now >= nextGeneratedCleanupAt) {
nextGeneratedCleanupAt = now + 60 * 60 * 1000
await this.purgeExpiredGeneratedFiles()
}
} catch (e: unknown) {
console.error('Error in asset cleanup job:', e)
}
+199 -1
View File
@@ -1,4 +1,5 @@
import { prisma } from '@shumai/db'
import * as advisoryLocks from '@shumai/db/src/advisory-lock'
import { setupTestDbHooks } from '@shumai/db/test'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { s3Service } from '../s3/s3'
@@ -32,6 +33,24 @@ function createMockWs() {
}
}
function reconcile(
service: RunnerService,
runnerId: string,
ws: RunnerSocket,
activeIds: string[] = [],
) {
service.handleMessage(
runnerId,
ws,
JSON.stringify({
type: 'reconcile',
environment: { platform: 'linux', shell: '/bin/sh' },
activeExecutions: activeIds.map((executionId) => ({ executionId, nextOffset: 0 })),
completedExecutions: [],
}),
)
}
describe('RunnerService', () => {
setupTestDbHooks()
@@ -49,11 +68,48 @@ describe('RunnerService', () => {
teamId = team.id
})
afterEach(() => {
afterEach(async () => {
service.stopHeartbeatMonitor()
await service.stopServerTopology()
vi.restoreAllMocks()
})
it('allows replicas without runners and enforces a single server while runners exist', async () => {
const replica = new RunnerService()
try {
await service.startServerTopology()
await replica.startServerTopology()
await expect(service.createRunner(teamId, { name: 'Blocked' })).rejects.toMatchObject({
status: 409,
message: expect.stringContaining('single main Shumai server'),
})
expect(await service.listRunners(teamId)).toEqual([])
await replica.stopServerTopology()
const { runner } = await service.createRunner(teamId, { name: 'Exclusive runner' })
await expect(replica.startServerTopology()).rejects.toThrow('single main Shumai server')
await service.stopServerTopology()
await replica.startServerTopology()
await expect(service.startServerTopology()).rejects.toThrow('single main Shumai server')
const acquireLease = advisoryLocks.tryAcquireUpgradeableAdvisoryLock
vi.spyOn(advisoryLocks, 'tryAcquireUpgradeableAdvisoryLock').mockImplementationOnce(
async (lockKey, exclusive) => {
// The newcomer has already looked up the existing runner.
await replica.deleteRunner(teamId, runner.id)
return acquireLease(lockKey, exclusive)
},
)
await service.startServerTopology()
await expect(replica.createRunner(teamId, { name: 'Blocked again' })).rejects.toMatchObject({
status: 409,
})
} finally {
await replica.stopServerTopology()
}
})
describe('Key & workspace path helpers', () => {
it('keeps validated OS and shell metadata only for the current connection', async () => {
const { runner, key } = await service.createRunner(teamId, { name: 'Windows runner' })
@@ -201,6 +257,25 @@ describe('RunnerService', () => {
})
describe('WebSocket connection lifecycle & eviction', () => {
it('waits for reconciliation before reporting online or dispatching a command', async () => {
const { runner, key } = await service.createRunner(teamId, { name: 'Handshaking runner' })
const conn = createMockWs()
await service.registerConnection(
{ id: runner.id, teamId, name: runner.name, keyHash: hashRunnerKey(key) },
conn.ws,
)
expect(conn.lastSentJson()).toHaveProperty('type', 'welcome')
expect(service.isRunnerConnected(runner.id)).toBe(false)
await expect(service.exec(runner.id, { command: 'echo ready' })).rejects.toThrow(
'Runner is not connected',
)
expect(conn.sent).toHaveLength(1)
reconcile(service, runner.id, conn.ws)
expect(service.isRunnerConnected(runner.id)).toBe(true)
})
it('evicts existing connection with 4000 when a new connection arrives', async () => {
const { runner, key } = await service.createRunner(teamId, { name: 'Runner WS' })
const keyHash = hashRunnerKey(key)
@@ -211,6 +286,7 @@ describe('RunnerService', () => {
{ id: runner.id, teamId, name: runner.name, keyHash },
conn1.ws,
)
reconcile(service, runner.id, conn1.ws)
expect(service.isRunnerConnected(runner.id)).toBe(true)
await service.registerConnection(
@@ -221,6 +297,7 @@ describe('RunnerService', () => {
code: 4000,
reason: 'Replaced by a newer connection',
})
reconcile(service, runner.id, conn2.ws)
expect(service.isRunnerConnected(runner.id)).toBe(true)
})
@@ -294,6 +371,7 @@ describe('RunnerService', () => {
{ id: r2.runner.id, teamId, name: r2.runner.name, keyHash: hashRunnerKey(r2.key) },
conn2.ws,
)
reconcile(service, r2.runner.id, conn2.ws)
const online = await service.requireOnlineRunner(teamId, r2.runner.id)
expect(online.id).toBe(r2.runner.id)
})
@@ -307,6 +385,7 @@ describe('RunnerService', () => {
{ id: runner.id, teamId, name: runner.name, keyHash: hashRunnerKey(key) },
conn.ws,
)
reconcile(service, runner.id, conn.ws)
const chunks: Array<{ stream: string; text: string }> = []
const execPromise = service.exec(runner.id, {
@@ -383,6 +462,7 @@ describe('RunnerService', () => {
conn.ws,
)
reconcile(service, runner.id, conn.ws)
const streamed: string[] = []
const noCollectPromise = service.exec(runner.id, {
command: 'yes | head -n 100',
@@ -456,6 +536,7 @@ describe('RunnerService', () => {
{ id: runner.id, teamId, name: runner.name, keyHash: hashRunnerKey(key) },
conn.ws,
)
reconcile(service, runner.id, conn.ws)
const abortController = new AbortController()
const execPromise = service.exec(runner.id, {
@@ -489,6 +570,121 @@ describe('RunnerService', () => {
expect(res.aborted).toBe(true)
})
it.each([false, true])(
'replays cancellation on reconnect after notifying caller=%s',
async (reported) => {
const { runner, key } = await service.createRunner(teamId, {
name: 'Cancelled while offline',
})
const authenticated = {
id: runner.id,
teamId,
name: runner.name,
keyHash: hashRunnerKey(key),
}
const conn = createMockWs()
await service.registerConnection(authenticated, conn.ws)
reconcile(service, runner.id, conn.ws)
const abort = new AbortController()
const execution = service.exec(runner.id, { command: 'sleep 30', signal: abort.signal })
const { executionId } = conn.lastSentJson()
service.handleDisconnect(runner.id, conn.ws)
abort.abort()
if (reported) await expect(execution).resolves.toMatchObject({ aborted: true })
const reconnected = createMockWs()
await service.registerConnection(authenticated, reconnected.ws)
expect(reconnected.lastSentJson()).toMatchObject({ inFlightExecutionIds: [executionId] })
reconcile(service, runner.id, reconnected.ws, [executionId])
const forwarded = reconnected.lastSentJson()
service.handleMessage(
runner.id,
reconnected.ws,
JSON.stringify({
type: 'exec.result',
executionId,
exitCode: 0,
signal: null,
timedOut: false,
}),
)
expect(forwarded).toEqual({ type: 'exec.cancel', executionId })
await expect(execution).resolves.toMatchObject({ aborted: true })
},
)
it('restages a generated input that was removed from the runner workspace', async () => {
vi.spyOn(s3Service, 'headObject').mockResolvedValue({
key: 'temp/generated/example/input.jpg',
size: 100,
lastModified: new Date(),
contentType: 'image/jpeg',
eTag: 'fixture',
})
vi.spyOn(s3Service, 'presign').mockResolvedValue('https://signed.example.com/input.jpg')
const { runner, key } = await service.createRunner(teamId, { name: 'Generated file runner' })
const conn = createMockWs()
await service.registerConnection(
{ id: runner.id, teamId, name: runner.name, keyHash: hashRunnerKey(key) },
conn.ws,
)
reconcile(service, runner.id, conn.ws)
const destPath = '.pi/generated/example/input.jpg'
const acknowledgeDownload = () => {
const download = conn.lastSentJson()
expect(download).toMatchObject({ type: 'file.download', destPath })
service.handleMessage(
runner.id,
conn.ws,
JSON.stringify({
type: 'file.download.result',
executionId: download.executionId,
ok: true,
}),
)
}
const first = service.ensureGeneratedFileOnRunner(runner.id, destPath)
await vi.waitFor(() => expect(conn.lastSentJson()?.type).toBe('file.download'))
acknowledgeDownload()
await expect(first).resolves.toBe(destPath)
const second = service.ensureGeneratedFileOnRunner(runner.id, destPath)
const stat = conn.lastSentJson()
expect(stat).toMatchObject({ type: 'file.stat', srcPath: destPath })
service.handleMessage(
runner.id,
conn.ws,
JSON.stringify({
type: 'file.stat.result',
executionId: stat.executionId,
ok: true,
exists: false,
}),
)
await vi.waitFor(() => expect(conn.lastSentJson()?.type).toBe('file.download'))
acknowledgeDownload()
await expect(second).resolves.toBe(destPath)
const third = service.ensureGeneratedFileOnRunner(runner.id, destPath)
const present = conn.lastSentJson()
expect(present).toMatchObject({ type: 'file.stat', srcPath: destPath })
service.handleMessage(
runner.id,
conn.ws,
JSON.stringify({
type: 'file.stat.result',
executionId: present.executionId,
ok: true,
exists: true,
isFile: true,
}),
)
await expect(third).resolves.toBe(destPath)
expect(conn.lastSentJson()).toEqual(present)
})
it('handles downloadToRunner, uploadFromRunner, statOnRunner, and session caching', async () => {
vi.spyOn(s3Service, 'presign').mockResolvedValue('https://signed.example.com/file')
const { runner, key } = await service.createRunner(teamId, { name: 'File Runner' })
@@ -497,6 +693,7 @@ describe('RunnerService', () => {
{ id: runner.id, teamId, name: runner.name, keyHash: hashRunnerKey(key) },
conn.ws,
)
reconcile(service, runner.id, conn.ws)
// 1. downloadToRunner
const dlPromise = service.downloadToRunner(runner.id, {
@@ -616,6 +813,7 @@ describe('RunnerService', () => {
{ id: runner.id, teamId, name: runner.name, keyHash },
conn1.ws,
)
reconcile(service, runner.id, conn1.ws)
const execPromise = service.exec(runner.id, { command: 'long-job' })
const startMsg = conn1.lastSentJson()
+131 -102
View File
@@ -12,6 +12,7 @@ import {
RunnerFileUploadResultMessage,
RunnerInfo,
RunnerEnvironment,
RUNNER_REPLACED_CLOSE_CODE,
RunnerToServerMessage,
runnerToServerMessageSchema,
ServerToRunnerMessage,
@@ -26,6 +27,7 @@ import type {
RunnerExecResult,
RunnerSocket,
} from './types'
import { RunnerTopology } from './runner-topology'
const RUNNER_KEY_PREFIX = 'shm_rn_'
const DEFAULT_RECONNECT_GRACE_MS = 30_000
@@ -60,6 +62,8 @@ interface PendingExecOperation {
resolve: (result: RunnerExecResult) => void
reject: (err: Error) => void
timeoutTimer?: ReturnType<typeof setTimeout>
abortTimer?: ReturnType<typeof setTimeout>
cancelRequested?: boolean
cleanupSignal?: () => void
}
@@ -132,11 +136,23 @@ export function validateRunnerWorkspacePath(targetPath: string): string {
export class RunnerService {
private sessions = new Map<string, ActiveRunnerSession>()
private closedSockets = new WeakSet<RunnerSocket>()
private pendingOps = new Map<string, PendingOperation>()
private disconnectGraceTimers = new Map<string, ReturnType<typeof setTimeout>>()
private credentialRevisions = new Map<string, number>()
private heartbeatTimer: ReturnType<typeof setInterval> | null = null
private reconnectGraceMs = DEFAULT_RECONNECT_GRACE_MS
private serverTopology?: RunnerTopology
async startServerTopology(): Promise<AbortSignal> {
this.serverTopology ??= await RunnerTopology.start()
return this.serverTopology.signal
}
async stopServerTopology(): Promise<void> {
await this.serverTopology?.close()
this.serverTopology = undefined
}
setReconnectGraceMs(ms: number): void {
this.reconnectGraceMs = ms
@@ -157,7 +173,7 @@ export class RunnerService {
this.handleDisconnect(runnerId, session.ws, 1001)
continue
}
this.sendToSession(session, { type: 'ping', ts: now })
if (session.environment) this.sendToSession(session, { type: 'ping', ts: now })
}
}, HEARTBEAT_INTERVAL_MS)
if (typeof this.heartbeatTimer.unref === 'function') {
@@ -173,7 +189,7 @@ export class RunnerService {
}
isRunnerConnected(runnerId: string): boolean {
return this.sessions.has(runnerId)
return this.getReadySession(runnerId) !== undefined
}
getRunnerEnvironment(runnerId: string): RunnerEnvironment | undefined {
@@ -203,6 +219,14 @@ export class RunnerService {
}
async createRunner(teamId: string, req: CreateRunnerRequest): Promise<CreateRunnerResponse> {
const create = () => this.persistRunner(teamId, req)
return this.serverTopology ? this.serverTopology.changeRunners(create, true) : create()
}
private async persistRunner(
teamId: string,
req: CreateRunnerRequest,
): Promise<CreateRunnerResponse> {
const name = req.name.trim()
const { key, keyPrefix, keyHash } = generateRunnerKey()
@@ -349,6 +373,11 @@ export class RunnerService {
}
async deleteRunner(teamId: string, runnerId: string): Promise<void> {
const remove = () => this.removeRunner(teamId, runnerId)
await (this.serverTopology ? this.serverTopology.changeRunners(remove) : remove())
}
private async removeRunner(teamId: string, runnerId: string): Promise<void> {
this.bumpCredentialRevision(runnerId)
await prisma.$transaction(async (tx) => {
@@ -472,6 +501,7 @@ export class RunnerService {
},
select: { id: true, teamId: true, name: true },
})
if (this.closedSockets.has(ws)) return
const currentRevision = this.credentialRevisions.get(runner.id) ?? 0
if (!verified || currentRevision !== startRevision) {
try {
@@ -482,19 +512,13 @@ export class RunnerService {
return
}
const existingTimer = this.disconnectGraceTimers.get(runner.id)
if (existingTimer) {
clearTimeout(existingTimer)
this.disconnectGraceTimers.delete(runner.id)
}
const existingSession = this.sessions.get(runner.id)
if (existingSession && existingSession.ws !== ws) {
logger.info({ runnerId: runner.id }, 'Evicting previous runner WebSocket connection')
// Remove from sessions before closing so handleDisconnect is a no-op for the old socket
this.sessions.delete(runner.id)
try {
existingSession.ws.close(4000, 'Replaced by a newer connection')
existingSession.ws.close(RUNNER_REPLACED_CLOSE_CODE, 'Replaced by a newer connection')
} catch {
// Ignore close errors
}
@@ -512,19 +536,6 @@ export class RunnerService {
this.sessions.set(runner.id, session)
this.startHeartbeatMonitor()
await prisma.runner
.update({
where: { id: runner.id },
data: { lastConnectedAt: new Date() },
})
.catch((err) => {
logger.warn({ err, runnerId: runner.id }, 'Failed to update runner lastConnectedAt')
})
if (this.sessions.get(runner.id) !== session) {
return
}
const inFlightExecutionIds = this.getInFlightExecutionIdsForRunner(runner.id)
this.sendToSession(session, {
type: 'welcome',
@@ -565,8 +576,13 @@ export class RunnerService {
case 'pong':
break
case 'reconcile':
session.environment = msg.environment
if (session.environment) break
this.handleReconcileMessage(session, msg)
session.environment = msg.environment
this.clearDisconnectGraceTimer(runnerId)
void prisma.runner
.update({ where: { id: runnerId }, data: { lastConnectedAt: new Date() } })
.catch((err) => logger.warn({ err, runnerId }, 'Failed to update runner lastConnectedAt'))
break
case 'exec.output':
this.handleExecOutputMessage(runnerId, msg)
@@ -587,6 +603,7 @@ export class RunnerService {
}
handleDisconnect(runnerId: string, ws: RunnerSocket, code?: number): void {
this.closedSockets.add(ws)
const session = this.sessions.get(runnerId)
if (!session || session.ws !== ws) {
return
@@ -608,6 +625,7 @@ export class RunnerService {
}
if (code === 1008 || this.reconnectGraceMs <= 0) {
this.clearDisconnectGraceTimer(runnerId)
this.failPendingOperationsForRunner(
runnerId,
new Error('Runner disconnected while operation was in progress'),
@@ -615,6 +633,7 @@ export class RunnerService {
return
}
if (this.disconnectGraceTimers.has(runnerId)) return
const timer = setTimeout(() => {
this.disconnectGraceTimers.delete(runnerId)
if (!this.isRunnerConnected(runnerId)) {
@@ -636,10 +655,7 @@ export class RunnerService {
// ----------------------------------------------------------------------
async exec(runnerId: string, options: RunnerExecOptions): Promise<RunnerExecResult> {
const session = this.sessions.get(runnerId)
if (!session) {
throw new Error('Runner is not connected')
}
const session = this.requireReadySession(runnerId)
if (options.signal?.aborted) {
throw new Error('Command aborted before execution started')
@@ -665,51 +681,22 @@ export class RunnerService {
const safetyTimeoutMs = Math.max((timeoutSec + 15) * 1000, 15_000)
const timeoutTimer = setTimeout(() => {
const pending = this.pendingOps.get(executionId)
if (!pending) return
if (pending?.kind !== 'exec') return
this.cleanupPendingOp(executionId)
const currentSession = this.sessions.get(runnerId)
const currentSession = this.getReadySession(runnerId)
if (currentSession) {
this.sendToSession(currentSession, { type: 'exec.cancel', executionId })
}
resolve({
executionId,
output: pending.kind === 'exec' ? this.readBufferedExecOutput(pending) : '',
output: this.readBufferedExecOutput(pending),
exitCode: null,
signal: 'SIGKILL',
timedOut: true,
aborted: false,
timedOut: !pending.cancelRequested,
aborted: pending.cancelRequested ?? false,
})
}, safetyTimeoutMs)
let cleanupSignal: (() => void) | undefined
if (options.signal) {
const onAbort = () => {
const currentSession = this.sessions.get(runnerId)
if (currentSession) {
this.sendToSession(currentSession, { type: 'exec.cancel', executionId })
}
// Give runner a short window to report exec.result, otherwise resolve as aborted
const abortTimer = setTimeout(() => {
const pending = this.pendingOps.get(executionId)
if (!pending) return
this.cleanupPendingOp(executionId)
resolve({
executionId,
output: pending.kind === 'exec' ? this.readBufferedExecOutput(pending) : '',
exitCode: null,
signal: 'SIGKILL',
timedOut: false,
aborted: true,
})
}, 2_000)
if (typeof abortTimer.unref === 'function') {
abortTimer.unref()
}
}
options.signal.addEventListener('abort', onAbort, { once: true })
cleanupSignal = () => options.signal?.removeEventListener('abort', onAbort)
}
const pendingOp: PendingExecOperation = {
kind: 'exec',
executionId,
@@ -724,7 +711,26 @@ export class RunnerService {
resolve,
reject,
timeoutTimer,
cleanupSignal,
}
if (options.signal) {
const signal = options.signal
const onAbort = () => {
pendingOp.cancelRequested = true
// Release the caller after a short grace period. Retain cancellation for reconnects.
pendingOp.abortTimer = setTimeout(() => {
this.reportCancelledExecution(pendingOp)
pendingOp.onOutput = undefined
pendingOp.collectOutput = false
pendingOp.outputChunks = []
pendingOp.bufferedOutputBytes = 0
}, 2_000)
pendingOp.abortTimer.unref?.()
const currentSession = this.getReadySession(runnerId)
if (currentSession)
this.sendToSession(currentSession, { type: 'exec.cancel', executionId })
}
signal.addEventListener('abort', onAbort, { once: true })
pendingOp.cleanupSignal = () => signal.removeEventListener('abort', onAbort)
}
this.pendingOps.set(executionId, pendingOp)
@@ -742,10 +748,7 @@ export class RunnerService {
timeoutMs?: number
},
): Promise<RunnerFileDownloadResultMessage> {
const session = this.sessions.get(runnerId)
if (!session) {
throw new Error('Runner is not connected')
}
const session = this.requireReadySession(runnerId)
const normalizedDestPath = validateRunnerWorkspacePath(options.destPath)
const executionId = ulid()
@@ -784,10 +787,7 @@ export class RunnerService {
srcPath: string,
timeoutMs = DEFAULT_FILE_OP_TIMEOUT_MS,
): Promise<RunnerFileStatResultMessage> {
const session = this.sessions.get(runnerId)
if (!session) {
throw new Error('Runner is not connected')
}
const session = this.requireReadySession(runnerId)
const normalizedSrcPath = validateRunnerWorkspacePath(srcPath)
const executionId = ulid()
@@ -826,10 +826,7 @@ export class RunnerService {
timeoutMs?: number
},
): Promise<RunnerFileUploadResultMessage> {
const session = this.sessions.get(runnerId)
if (!session) {
throw new Error('Runner is not connected')
}
const session = this.requireReadySession(runnerId)
const normalizedSrcPath = validateRunnerWorkspacePath(options.srcPath)
const executionId = ulid()
@@ -866,10 +863,7 @@ export class RunnerService {
runnerId: string,
skill: { id: string; assetId: string; hash: string },
): Promise<string> {
const session = this.sessions.get(runnerId)
if (!session) {
throw new Error('Runner is not connected')
}
const session = this.requireReadySession(runnerId)
const destPath = `.pi/skills/${skill.id}`
if (session.stagedSkills.get(skill.id) === skill.hash) {
@@ -910,10 +904,7 @@ export class RunnerService {
}
async ensureGeneratedFileOnRunner(runnerId: string, virtualPath: string): Promise<string> {
const session = this.sessions.get(runnerId)
if (!session) {
throw new Error('Runner is not connected')
}
const session = this.requireReadySession(runnerId)
const normalizedPath = validateRunnerWorkspacePath(virtualPath)
if (!normalizedPath.startsWith('.pi/generated/')) {
@@ -923,7 +914,10 @@ export class RunnerService {
}
if (session.stagedGeneratedFiles.has(normalizedPath)) {
return normalizedPath
const stat = await this.statOnRunner(runnerId, normalizedPath)
if (stat.ok && stat.exists && stat.isFile && this.getReadySession(runnerId) === session) {
return normalizedPath
}
}
// Virtual path .pi/generated/<ulid>/<filename> maps to storage key temp/generated/<ulid>/<filename>
@@ -961,34 +955,54 @@ export class RunnerService {
// Internal Helpers
// ----------------------------------------------------------------------
private getReadySession(runnerId: string): ActiveRunnerSession | undefined {
const session = this.sessions.get(runnerId)
return session?.environment ? session : undefined
}
private requireReadySession(runnerId: string): ActiveRunnerSession {
const session = this.getReadySession(runnerId)
if (!session) throw new Error('Runner is not connected')
return session
}
private handleReconcileMessage(
session: ActiveRunnerSession,
msg: Extract<RunnerToServerMessage, { type: 'reconcile' }>,
): void {
const completedIds = new Set<string>()
for (const completed of msg.completedExecutions) {
completedIds.add(completed.executionId)
this.handleExecResultMessage(session.runnerId, completed)
}
const activeIds = new Set(msg.activeExecutions.map((a) => a.executionId))
for (const executionId of activeIds) {
const pending = this.pendingOps.get(executionId)
if (
pending?.runnerId !== session.runnerId ||
pending.kind !== 'exec' ||
pending.cancelRequested
) {
// A timeout or cancellation may have occurred after the welcome snapshot.
this.sendToSession(session, { type: 'exec.cancel', executionId })
}
}
for (const [executionId, pending] of this.pendingOps.entries()) {
if (pending.runnerId !== session.runnerId) continue
if (completedIds.has(executionId) || activeIds.has(executionId)) {
continue
}
if (pending.kind === 'exec') {
if (pending.kind !== 'exec') {
// Re-send idempotent file control operations on reconnect.
this.sendToSession(session, pending.message)
} else if (!activeIds.has(executionId)) {
this.cleanupPendingOp(executionId)
if (pending.cancelRequested) {
this.reportCancelledExecution(pending)
continue
}
pending.reject(
new Error(
`Runner reconnected without active execution ${executionId} (runner process may have restarted)`,
),
)
} else {
// Re-send idempotent file control operations on reconnect
this.sendToSession(session, pending.message)
}
}
}
@@ -1031,8 +1045,8 @@ export class RunnerService {
output: this.readBufferedExecOutput(pending),
exitCode: msg.exitCode,
signal: msg.signal,
timedOut: msg.timedOut,
aborted: msg.aborted ?? false,
timedOut: msg.timedOut && !pending.cancelRequested,
aborted: pending.cancelRequested || (msg.aborted ?? false),
fullOutputPath: msg.fullOutputPath,
error: msg.error,
})
@@ -1133,11 +1147,7 @@ export class RunnerService {
private revokeActiveSession(runnerId: string, reason: string): void {
this.bumpCredentialRevision(runnerId)
const timer = this.disconnectGraceTimers.get(runnerId)
if (timer) {
clearTimeout(timer)
this.disconnectGraceTimers.delete(runnerId)
}
this.clearDisconnectGraceTimer(runnerId)
const session = this.sessions.get(runnerId)
if (session) {
@@ -1152,6 +1162,24 @@ export class RunnerService {
this.failPendingOperationsForRunner(runnerId, new Error(reason))
}
private clearDisconnectGraceTimer(runnerId: string): void {
const timer = this.disconnectGraceTimers.get(runnerId)
if (!timer) return
clearTimeout(timer)
this.disconnectGraceTimers.delete(runnerId)
}
private reportCancelledExecution(pending: PendingExecOperation): void {
pending.resolve({
executionId: pending.executionId,
output: this.readBufferedExecOutput(pending),
exitCode: null,
signal: 'SIGKILL',
timedOut: false,
aborted: true,
})
}
private failPendingOperationsForRunner(runnerId: string, error: Error): void {
for (const [executionId, pending] of this.pendingOps.entries()) {
if (pending.runnerId === runnerId) {
@@ -1168,8 +1196,9 @@ export class RunnerService {
if (pending.timeoutTimer) {
clearTimeout(pending.timeoutTimer)
}
if (pending.kind === 'exec' && pending.cleanupSignal) {
pending.cleanupSignal()
if (pending.kind === 'exec') {
clearTimeout(pending.abortTimer)
pending.cleanupSignal?.()
}
}
@@ -0,0 +1,76 @@
import { HTTPException } from 'hono/http-exception'
import { prisma } from '@shumai/db'
import { tryAcquireUpgradeableAdvisoryLock } from '@shumai/db/src/advisory-lock'
import { logger } from '@shumai/core/src/logger'
const RUNNER_TOPOLOGY_KEY = 'shumai:runner-topology'
const SINGLE_SERVER_REQUIRED =
'Agent runners require a single main Shumai server per database. Stop the other main server replicas.'
async function hasConfiguredRunners() {
return (await prisma.runner.findFirst({ select: { id: true } })) !== null
}
/** Each main server holds a shared lease. A configured runner requires an exclusive lease. */
export class RunnerTopology {
private mutations = Promise.resolve()
private constructor(
private readonly lease: NonNullable<
Awaited<ReturnType<typeof tryAcquireUpgradeableAdvisoryLock>>
>,
) {}
static async start() {
const configured = await hasConfiguredRunners()
let lease = await tryAcquireUpgradeableAdvisoryLock(RUNNER_TOPOLOGY_KEY, configured)
if (!lease && configured) {
// If the last runner was just deleted, the incumbent may now hold a shared lease.
lease = await tryAcquireUpgradeableAdvisoryLock(RUNNER_TOPOLOGY_KEY, false)
}
if (!lease) throw new Error(SINGLE_SERVER_REQUIRED)
try {
// A concurrently stopping server may have changed the catalog.
if (await hasConfiguredRunners()) {
if (!(await lease.tryUpgrade())) throw new Error(SINGLE_SERVER_REQUIRED)
} else {
await lease.downgrade()
}
return new RunnerTopology(lease)
} catch (error) {
await lease.release()
throw error
}
}
get signal() {
return this.lease.signal
}
/** Serialize runner creation and deletion until the exclusive lease can be safely downgraded. */
async changeRunners<T>(operation: () => Promise<T>, adding = false): Promise<T> {
const previous = this.mutations
const finished = Promise.withResolvers<void>()
this.mutations = finished.promise
await previous
try {
if (adding && !(await this.lease.tryUpgrade())) {
throw new HTTPException(409, { message: SINGLE_SERVER_REQUIRED })
}
return await operation()
} finally {
try {
if (this.lease.exclusive && !(await hasConfiguredRunners())) await this.lease.downgrade()
} catch (error) {
logger.warn({ err: error }, 'Failed to downgrade the runner server lease')
} finally {
finished.resolve()
}
}
}
async close() {
await this.mutations
await this.lease.release()
}
}
+44
View File
@@ -717,6 +717,50 @@ describe('S3Service implementations', () => {
expect(keysAll.length).toBe(3)
})
it('lists a nested prefix without scanning unrelated directories', async () => {
await localS3.putObject('test-bucket', 'temp/generated/one.jpg', 'one', 3)
await localS3.putObject('test-bucket', 'files/unrelated.txt', 'abc', 3)
const readDirectory = vi.spyOn(fs.promises, 'readdir')
try {
await expect(localS3.listObjects('test-bucket', 'temp/generated/')).resolves.toEqual([
'temp/generated/one.jpg',
])
expect(readDirectory.mock.calls.map(([directory]) => directory)).toEqual([
path.join(TEST_BASE_PATH, 'test-bucket', 'temp', 'generated'),
])
readDirectory.mockClear()
await expect(localS3.listObjects('test-bucket', 'temp/missing/')).resolves.toEqual([])
expect(readDirectory.mock.calls.map(([directory]) => directory)).not.toContain(
path.join(TEST_BASE_PATH, 'test-bucket'),
)
await expect(localS3.listObjects('test-bucket', 'temp/generated/on')).resolves.toEqual([
'temp/generated/one.jpg',
])
} finally {
readDirectory.mockRestore()
}
})
it('continues listing when a file disappears during a local prefix scan', async () => {
const removedKey = 'temp/generated/removed.jpg'
const remainingKey = 'temp/generated/remaining.jpg'
await localS3.putObject('test-bucket', removedKey, 'old', 3)
await localS3.putObject('test-bucket', remainingKey, 'new', 3)
const stat = fs.promises.stat
const removedPath = path.join(TEST_BASE_PATH, 'test-bucket', removedKey)
const statSpy = vi.spyOn(fs.promises, 'stat').mockImplementation(async (target, options) => {
if (target === removedPath) await fs.promises.unlink(removedPath)
return stat(target, options)
})
try {
await expect(localS3.listObjects('test-bucket', 'temp/generated/')).resolves.toEqual([
remainingKey,
])
} finally {
statSpy.mockRestore()
}
})
it('should handle headObject correctly', async () => {
await localS3.putObject('test-bucket', 'test.txt', 'hello world', 11)
const head = await localS3.headObject('test-bucket', 'test.txt')
+23 -6
View File
@@ -1012,16 +1012,33 @@ export class LocalStorageService implements S3Service {
async listObjects(bucket: string, prefix: string): Promise<string[]> {
const bucketDir = path.join(this.basePath, bucket)
const keys: string[] = []
if (!fs.existsSync(bucketDir)) {
const directoryPrefix = prefix.slice(0, prefix.lastIndexOf('/') + 1)
const directory = path.join(bucketDir, directoryPrefix.slice(0, -1))
const relativeDirectory = path.relative(bucketDir, directory).split(path.sep).join('/')
if (
directoryPrefix &&
(!relativeDirectory ||
relativeDirectory === '..' ||
relativeDirectory.startsWith('../') ||
relativeDirectory + '/' !== directoryPrefix)
)
return keys
}
const ignoreMissing = (error: unknown) => {
if (
error instanceof Error &&
'code' in error &&
(error.code === 'ENOENT' || error.code === 'ENOTDIR')
)
return undefined
throw error
}
const walkDir = async (currentDir: string) => {
const files = await fs.promises.readdir(currentDir)
const files = (await fs.promises.readdir(currentDir).catch(ignoreMissing)) ?? []
for (const file of files) {
const fullPath = path.join(currentDir, file)
const stat = await fs.promises.stat(fullPath)
const stat = await fs.promises.stat(fullPath).catch(ignoreMissing)
if (!stat) continue
if (stat.isDirectory()) {
await walkDir(fullPath)
@@ -1037,7 +1054,7 @@ export class LocalStorageService implements S3Service {
}
}
await walkDir(bucketDir)
await walkDir(directory)
return keys
}
+65
View File
@@ -39,3 +39,68 @@ export async function acquireAdvisoryLock(key: string, signal?: AbortSignal) {
},
}
}
/** Hold a shared server lease, with a non-blocking upgrade to exclusive ownership. */
export async function tryAcquireUpgradeableAdvisoryLock(key: string, exclusive: boolean) {
const client = await locks.connect()
const lost = new AbortController()
const onError = (error: Error) => lost.abort(error)
client.on('error', onError)
const tryExclusive = async () => {
const result = await client.query<{ acquired: boolean }>(
'SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired',
[key],
)
return result.rows[0]?.acquired === true
}
let acquired = false
try {
if (exclusive && !(await tryExclusive())) return null
const result = await client.query<{ acquired: boolean }>(
'SELECT pg_try_advisory_lock_shared(hashtextextended($1, 0)) AS acquired',
[key],
)
lost.signal.throwIfAborted()
acquired = result.rows[0]?.acquired === true
if (!acquired) return null
} finally {
if (!acquired) {
client.off('error', onError)
client.release(true)
}
}
let released = false
const assertHeld = () => {
if (released) throw new Error('The server lease has been released')
lost.signal.throwIfAborted()
}
return {
signal: lost.signal,
get exclusive() {
return exclusive
},
async tryUpgrade() {
assertHeld()
if (!exclusive) exclusive = await tryExclusive()
return exclusive
},
async downgrade() {
assertHeld()
if (!exclusive) return
await client.query('SELECT pg_advisory_unlock(hashtextextended($1, 0))', [key])
exclusive = false
},
async release() {
if (released) return
released = true
try {
if (!lost.signal.aborted) await client.query('SELECT pg_advisory_unlock_all()')
} finally {
client.off('error', onError)
client.release(true)
}
},
}
}
+2
View File
@@ -48,6 +48,8 @@ export type RotateRunnerKeyResponse = z.infer<typeof rotateRunnerKeyResponseSche
// WebSocket Wire Protocol Schemas
// ----------------------------------------------------------------------
export const RUNNER_REPLACED_CLOSE_CODE = 4000
export const runnerWelcomeMessageSchema = z.object({
type: z.literal('welcome'),
runnerId: z.string(),
+10
View File
@@ -262,6 +262,16 @@ describe('Workflow E2E - read_asset', () => {
{ id: runner.id, teamId: team.id, name: runner.name, keyHash: hashRunnerKey(key) },
runnerSocket,
)
runnerService.handleMessage(
runner.id,
runnerSocket,
JSON.stringify({
type: 'reconcile',
environment: { platform: 'linux', shell: '/bin/sh' },
activeExecutions: [],
completedExecutions: [],
}),
)
const video = await createFileAsset(
project.id,
+2 -2
View File
@@ -1703,10 +1703,10 @@
"task_type_restore_asset": "Restore Asset",
"agent_runners": "Agent Runners",
"agent_runners_settings": "Agent Runners",
"runners_description": "Register and manage remote runners that execute shell commands and skills for your agents.",
"runners_description": "Register and manage remote runners. Everyone with shell access to a runner can read files in its workspace, including files from other projects.",
"runners": "Runners",
"create_runner": "Create Runner",
"create_runner_description": "Register a new runner for your team. You will receive a one-time runner key to connect from shumai-cli.",
"create_runner_description": "Register a runner for your team and get a one-time key for shumai-cli. Share the runner only with users who may read all files in its workspace.",
"runner_name": "Runner Name",
"runner_name_placeholder": "e.g. Mac Mini Runner",
"runner_description": "Description (Optional)",
+2 -2
View File
@@ -1703,10 +1703,10 @@
"task_type_restore_asset": "恢复归档资产",
"agent_runners": "Agent 运行器",
"agent_runners_settings": "Agent 运行器设置",
"runners_description": "注册并管理为 Agent 执行命令和技能的远程运行器。",
"runners_description": "注册并管理远程运行器。有权在同一运行器执行命令的用户可以读取其工作区中的文件,包括其他项目留下的文件。",
"runners": "运行器",
"create_runner": "创建运行器",
"create_runner_description": "为团队注册一个新的运行器。创建后将生成一次性运行器密钥,用于通过 shumai-cli 连接。",
"create_runner_description": "为团队注册运行器并获取用于 shumai-cli 的一次性密钥。请仅在有权读取工作区内所有文件的用户之间共享运行器。",
"runner_name": "运行器名称",
"runner_name_placeholder": "例如:Mac Mini 运行器",
"runner_description": "描述(可选)",