mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-22 17:48:25 +00:00
* refactor(agent)!: migrate sessions to Pi Durable Replace patched AgentHarness storage with per-session SQLite and PostgreSQL commit mirroring. Migrate legacy history automatically and run agent queues with the main server. BREAKING CHANGE: Remove standalone agent workers and return native Pi entry records from chat history APIs. * fix(agent): constrain session log dialog scrolling Keep the header visible and wrap long output while log entries and expanded records scroll within the dialog. Cover desktop and narrow viewports through the dashboard Sessions flow. * fix(agent): hide entry IDs from session log card headers Retain internal entry IDs in expanded raw records for debugging. Verify that card headers omit the IDs while raw JSON preserves them. * test(e2e): handle browser wheel limits in session log checks Verify that a normal wheel event moves the viewport before explicitly scrolling to the final record. Keep the dialog bounds, fixed header, wrapping, and expanded-record assertions. * fix(agent): select model providers without inheriting unrelated auth * fix(webui): prevent resize feedback when toggling agent panel * test(agent): reproduce concurrent comment session races * fix(agent): serialize comment sessions and deduplicate history * docs(changelog): consolidate Pi Durable changes into one entry
574 lines
16 KiB
TypeScript
574 lines
16 KiB
TypeScript
import tailwindPlugin from 'bun-plugin-tailwind'
|
||
import { existsSync, readFileSync, readdirSync } from 'node:fs'
|
||
import { cp, mkdir, readdir, rm, writeFile } from 'node:fs/promises'
|
||
import path from 'node:path'
|
||
|
||
console.log('\n🚀 Starting NPM packaging build process for all applications...\n')
|
||
|
||
const rootDir = process.cwd()
|
||
const distDir = path.join(rootDir, 'dist')
|
||
|
||
// 1. Clean previous build
|
||
if (existsSync(distDir)) {
|
||
console.log(`🗑️ Cleaning previous build at ${distDir}`)
|
||
await rm(distDir, { recursive: true, force: true })
|
||
}
|
||
await mkdir(distDir, { recursive: true })
|
||
|
||
const dummyRunnerPath = path.join(distDir, 'dummy-runner.ts')
|
||
await writeFile(dummyRunnerPath, '// Shumai Runner Stub\n', 'utf-8')
|
||
|
||
const rootPackageJson = JSON.parse(await Bun.file('package.json').text())
|
||
const version = rootPackageJson.version || '0.1.0'
|
||
|
||
const commonExternal = [
|
||
'@dbos-inc/dbos-sdk',
|
||
'@prisma/client',
|
||
'@prisma/adapter-pg',
|
||
'pg',
|
||
'sharp',
|
||
'pino',
|
||
'pino-pretty',
|
||
'pdfkit',
|
||
'prisma',
|
||
'zod',
|
||
// Exclude from bundling because it relies on native binaries/dynamic loading
|
||
'exiftool-vendored',
|
||
]
|
||
|
||
// Extract exact dependency versions from workspace package.json files
|
||
console.log('🔍 Extracting exact dependency versions for runtime packages...')
|
||
const dependencyVersions: Record<string, string> = {}
|
||
|
||
// Find all workspace package.json files
|
||
const packageJsonPaths = ['package.json']
|
||
const workspaceDirs = ['packages', 'apps']
|
||
for (const dir of workspaceDirs) {
|
||
if (existsSync(dir)) {
|
||
const subdirs = readdirSync(dir)
|
||
for (const subdir of subdirs) {
|
||
const p = path.join(dir, subdir, 'package.json')
|
||
if (existsSync(p)) {
|
||
packageJsonPaths.push(p)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
for (const dep of commonExternal) {
|
||
let foundVersion: string | null = null
|
||
for (const p of packageJsonPaths) {
|
||
try {
|
||
const content = readFileSync(p, 'utf8')
|
||
const pkgJson = JSON.parse(content)
|
||
const version = pkgJson.dependencies?.[dep] || pkgJson.devDependencies?.[dep]
|
||
if (version) {
|
||
// Strip any leading range specifiers like ^ or ~ to get the base version
|
||
foundVersion = version.replace(/^[\^~]/, '')
|
||
break
|
||
}
|
||
} catch {
|
||
// ignore read/parse errors
|
||
}
|
||
}
|
||
|
||
if (foundVersion) {
|
||
dependencyVersions[dep] = foundVersion
|
||
console.log(` ✅ Found ${dep}@${foundVersion}`)
|
||
} else {
|
||
console.warn(` ⚠️ Warning: Could not find exact version for ${dep}`)
|
||
}
|
||
}
|
||
|
||
const targets = [
|
||
{ platform: 'darwin', arch: 'arm64', bunTarget: 'bun-darwin-arm64', suffix: '' },
|
||
{ platform: 'darwin', arch: 'x64', bunTarget: 'bun-darwin-x64', suffix: '' },
|
||
{ platform: 'linux', arch: 'arm64', bunTarget: 'bun-linux-arm64', suffix: '' },
|
||
{ platform: 'linux', arch: 'x64', bunTarget: 'bun-linux-x64', suffix: '' },
|
||
{ platform: 'win32', arch: 'arm64', bunTarget: 'bun-windows-arm64', suffix: '.exe' },
|
||
{ platform: 'win32', arch: 'x64', bunTarget: 'bun-windows-x64', suffix: '.exe' },
|
||
]
|
||
|
||
// Function to compile a single application binary
|
||
async function compileApp({
|
||
name,
|
||
entrypoint,
|
||
outdir,
|
||
binaryName,
|
||
bunTarget,
|
||
plugins = [],
|
||
external = commonExternal,
|
||
}: {
|
||
name: string
|
||
entrypoint: string
|
||
outdir: string
|
||
binaryName: string
|
||
bunTarget?: string
|
||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||
plugins?: any[]
|
||
external?: string[]
|
||
}) {
|
||
console.log(`\n📦 Building binary for ${name}${bunTarget ? ` (${bunTarget})` : ''}...`)
|
||
await mkdir(outdir, { recursive: true })
|
||
|
||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||
const compileOptions: Record<string, any> = {
|
||
autoloadPackageJson: true,
|
||
}
|
||
if (bunTarget) {
|
||
compileOptions.target = bunTarget
|
||
}
|
||
|
||
const result = await Bun.build({
|
||
entrypoints: [entrypoint],
|
||
root: '.',
|
||
target: 'bun',
|
||
outdir: outdir,
|
||
minify: true,
|
||
compile: compileOptions,
|
||
plugins,
|
||
external,
|
||
define: {
|
||
'process.env.NODE_ENV': JSON.stringify('production'),
|
||
},
|
||
publicPath: '/',
|
||
})
|
||
|
||
if (!result.success) {
|
||
console.error(`❌ Binary compilation failed for ${name}!`)
|
||
for (const log of result.logs) {
|
||
console.error(log)
|
||
}
|
||
process.exit(1)
|
||
}
|
||
|
||
// Rename generated binary dynamically by finding the output file in outdir
|
||
const finalFile = path.join(outdir, binaryName)
|
||
const files = await readdir(outdir)
|
||
const generatedFilename = files.find((f) => f !== binaryName)
|
||
|
||
if (generatedFilename) {
|
||
const generatedPath = path.join(outdir, generatedFilename)
|
||
console.log(`➡️ Moving binary to final location: ${finalFile}`)
|
||
await cp(generatedPath, finalFile)
|
||
await rm(generatedPath)
|
||
} else if (!existsSync(finalFile)) {
|
||
console.error(`❌ Could not find compiled binary in ${outdir}`)
|
||
process.exit(1)
|
||
}
|
||
}
|
||
|
||
// Function to build platform-specific sub-packages for an application
|
||
async function buildPlatformPackages(
|
||
appName: string,
|
||
entrypoint: string,
|
||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||
plugins: any[] = [],
|
||
external?: string[],
|
||
) {
|
||
for (const target of targets) {
|
||
const dirName = `${appName}-${target.platform}-${target.arch}`
|
||
const npmName = `@shumai-one/${appName}-${target.platform}-${target.arch}`
|
||
const pkgOutDir = path.join(distDir, dirName)
|
||
const binOutDir = path.join(pkgOutDir, 'bin')
|
||
const binaryName = appName + target.suffix
|
||
|
||
await compileApp({
|
||
name: dirName,
|
||
entrypoint,
|
||
outdir: binOutDir,
|
||
binaryName,
|
||
bunTarget: target.bunTarget,
|
||
plugins,
|
||
external,
|
||
})
|
||
|
||
// Generate package.json for platform-specific package
|
||
const platformPackageJson = {
|
||
name: npmName,
|
||
version,
|
||
description: `${target.platform} ${target.arch} binary for ${appName}`,
|
||
os: [target.platform],
|
||
cpu: [target.arch],
|
||
repository: rootPackageJson.repository,
|
||
}
|
||
|
||
await writeFile(
|
||
path.join(pkgOutDir, 'package.json'),
|
||
JSON.stringify(platformPackageJson, null, 2),
|
||
'utf-8',
|
||
)
|
||
}
|
||
}
|
||
|
||
// Function to build main wrapper package
|
||
// Function to build main wrapper package
|
||
async function buildMainPackage({
|
||
appName,
|
||
description,
|
||
hasPrisma,
|
||
entrypoint,
|
||
plugins = [],
|
||
extraDependencies = {},
|
||
external = commonExternal,
|
||
}: {
|
||
appName: string
|
||
description: string
|
||
hasPrisma: boolean
|
||
entrypoint: string
|
||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||
plugins?: any[]
|
||
extraDependencies?: Record<string, string>
|
||
external?: string[]
|
||
}) {
|
||
console.log(`\n📄 Packaging wrapper app ${appName}...`)
|
||
const mainOutDir = path.join(distDir, appName)
|
||
const binOutDir = path.join(mainOutDir, 'bin')
|
||
await mkdir(binOutDir, { recursive: true })
|
||
|
||
// 1. Bundle the JS application code for wrapper
|
||
console.log(`📦 Bundling JS application code for wrapper ${appName}...`)
|
||
const appJsName = `${appName}-app.js`
|
||
|
||
const buildResult = await Bun.build({
|
||
entrypoints: [entrypoint],
|
||
root: '.',
|
||
target: 'bun',
|
||
outdir: binOutDir,
|
||
minify: true,
|
||
splitting: true,
|
||
naming: `${appName}-app.[ext]`,
|
||
plugins,
|
||
external,
|
||
define: {
|
||
'process.env.NODE_ENV': JSON.stringify('production'),
|
||
},
|
||
})
|
||
|
||
if (!buildResult.success) {
|
||
console.error(`❌ JS bundling failed for ${appName}!`)
|
||
for (const log of buildResult.logs) {
|
||
console.error(log)
|
||
}
|
||
process.exit(1)
|
||
}
|
||
|
||
// Confirm that the bundle exists
|
||
const finalJsFile = path.join(binOutDir, appJsName)
|
||
if (!existsSync(finalJsFile)) {
|
||
console.error(`❌ Could not find compiled JS bundle at ${finalJsFile}`)
|
||
process.exit(1)
|
||
}
|
||
|
||
// Write JS wrapper script
|
||
const wrapperContent = `#!/usr/bin/env node
|
||
import { spawn } from 'node:child_process'
|
||
import { join, dirname } from 'node:path'
|
||
import { fileURLToPath } from 'node:url'
|
||
import { existsSync, readFileSync } from 'node:fs'
|
||
|
||
const __dirname = dirname(fileURLToPath(import.meta.url))
|
||
const platform = process.platform
|
||
const arch = process.arch
|
||
|
||
let pkgName = ''
|
||
let binaryName = ''
|
||
|
||
if (platform === 'darwin' && arch === 'arm64') {
|
||
pkgName = '@shumai-one/${appName}-darwin-arm64'
|
||
binaryName = '${appName}'
|
||
} else if (platform === 'darwin' && arch === 'x64') {
|
||
pkgName = '@shumai-one/${appName}-darwin-x64'
|
||
binaryName = '${appName}'
|
||
} else if (platform === 'linux' && arch === 'arm64') {
|
||
pkgName = '@shumai-one/${appName}-linux-arm64'
|
||
binaryName = '${appName}'
|
||
} else if (platform === 'linux' && arch === 'x64') {
|
||
pkgName = '@shumai-one/${appName}-linux-x64'
|
||
binaryName = '${appName}'
|
||
} else if (platform === 'win32' && arch === 'arm64') {
|
||
pkgName = '@shumai-one/${appName}-win32-arm64'
|
||
binaryName = '${appName}.exe'
|
||
} else if (platform === 'win32' && arch === 'x64') {
|
||
pkgName = '@shumai-one/${appName}-win32-x64'
|
||
binaryName = '${appName}.exe'
|
||
} else {
|
||
console.error(\`Unsupported platform/architecture: \${platform}/\${arch}\`)
|
||
process.exit(1)
|
||
}
|
||
|
||
const pathsToTry = [
|
||
join(__dirname, '..', '..', '..', pkgName, 'bin', binaryName), // Flat node_modules
|
||
join(__dirname, '..', 'node_modules', pkgName, 'bin', binaryName), // Nested node_modules
|
||
]
|
||
|
||
let binaryPath = ''
|
||
for (const p of pathsToTry) {
|
||
if (existsSync(p)) {
|
||
binaryPath = p
|
||
break
|
||
}
|
||
}
|
||
|
||
if (!binaryPath) {
|
||
console.error(\`Could not find binary for platform package "\${pkgName}". Please ensure optional dependencies are installed.\`)
|
||
process.exit(1)
|
||
}
|
||
|
||
const appJsPath = join(__dirname, '${appName}-app.js')
|
||
|
||
let activeProcess = null
|
||
|
||
function startApp() {
|
||
activeProcess = spawn(binaryPath, ['run', appJsPath, ...process.argv.slice(2)], {
|
||
stdio: 'inherit',
|
||
env: {
|
||
...process.env,
|
||
BUN_BE_BUN: '1',
|
||
},
|
||
})
|
||
|
||
activeProcess.on('close', (code) => {
|
||
process.exit(code ?? 0)
|
||
})
|
||
|
||
activeProcess.on('error', (err) => {
|
||
console.error(\`Failed to start child process:\`, err)
|
||
process.exit(1)
|
||
})
|
||
}
|
||
|
||
// Simple .env parser to load env files from process.cwd()
|
||
function loadEnv() {
|
||
const envFiles = ['.env.production.local', '.env.local', '.env.production', '.env']
|
||
for (const file of envFiles) {
|
||
const envPath = join(process.cwd(), file)
|
||
if (existsSync(envPath)) {
|
||
try {
|
||
const content = readFileSync(envPath, 'utf8')
|
||
const lines = content.split(/\\r?\\n/)
|
||
for (const line of lines) {
|
||
const trimmed = line.trim()
|
||
if (!trimmed || trimmed.startsWith('#')) continue
|
||
const match = trimmed.match(/^([^=]+)=(.*)$/)
|
||
if (match) {
|
||
const key = match[1].trim()
|
||
let value = match[2].trim()
|
||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
||
value = value.substring(1, value.length - 1)
|
||
}
|
||
if (process.env[key] === undefined) {
|
||
process.env[key] = value
|
||
}
|
||
}
|
||
}
|
||
} catch (err) {
|
||
console.error('⚠️ [shumai] Error reading env file:', err)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Load env files on startup
|
||
loadEnv()
|
||
|
||
console.log(\`ℹ️ [${appName}] Running from CWD: \${process.cwd()}\`)
|
||
console.log(\`ℹ️ [${appName}] DATABASE_URL is: \${process.env.DATABASE_URL ? '(defined)' : '(undefined)'}\`)
|
||
|
||
const prismaSchemaPath = join(__dirname, '..', 'prisma', 'schema.prisma')
|
||
const isUtilityCommand = process.argv.includes('stop') || process.argv.includes('logs')
|
||
if (existsSync(prismaSchemaPath) && !isUtilityCommand) {
|
||
console.log('🔄 Running database migrations...')
|
||
activeProcess = spawn(
|
||
binaryPath,
|
||
['run', 'prisma', 'migrate', 'deploy', '--schema', './prisma/schema.prisma'],
|
||
{
|
||
stdio: 'inherit',
|
||
cwd: join(__dirname, '..'),
|
||
env: {
|
||
...process.env,
|
||
BUN_BE_BUN: '1',
|
||
},
|
||
}
|
||
)
|
||
|
||
activeProcess.on('close', (code) => {
|
||
if (code !== 0) {
|
||
console.error(\`❌ Database migration failed with code \${code}. Exiting.\`)
|
||
process.exit(code ?? 1)
|
||
}
|
||
startApp()
|
||
})
|
||
|
||
activeProcess.on('error', (err) => {
|
||
console.error('❌ Failed to run database migrations:', err)
|
||
process.exit(1)
|
||
})
|
||
} else {
|
||
startApp()
|
||
}
|
||
|
||
// Forward kill signals to the active child process
|
||
const cleanupAndExit = (signal) => {
|
||
if (activeProcess && !activeProcess.killed) {
|
||
activeProcess.kill(signal)
|
||
} else {
|
||
process.exit(0)
|
||
}
|
||
}
|
||
|
||
process.on('SIGINT', () => cleanupAndExit('SIGINT'))
|
||
process.on('SIGTERM', () => cleanupAndExit('SIGTERM'))
|
||
process.on('SIGQUIT', () => cleanupAndExit('SIGQUIT'))
|
||
|
||
`
|
||
|
||
const wrapperPath = path.join(binOutDir, `${appName}.js`)
|
||
await writeFile(wrapperPath, wrapperContent, { encoding: 'utf-8', mode: 0o755 })
|
||
|
||
// Copy prisma directory if needed
|
||
if (hasPrisma) {
|
||
console.log(`📂 Copying database files for ${appName}...`)
|
||
const dbDest = path.join(mainOutDir, 'prisma')
|
||
await mkdir(dbDest, { recursive: true })
|
||
await cp('./packages/db/prisma/schema.prisma', path.join(dbDest, 'schema.prisma'))
|
||
if (existsSync('./packages/db/prisma/migrations')) {
|
||
await cp('./packages/db/prisma/migrations', path.join(dbDest, 'migrations'), {
|
||
recursive: true,
|
||
})
|
||
}
|
||
|
||
// Generate prisma.config.ts in the package root
|
||
const localConfigContent = `import { defineConfig } from 'prisma/config'
|
||
|
||
export default defineConfig({
|
||
schema: './prisma/schema.prisma',
|
||
migrations: {
|
||
path: './prisma/migrations',
|
||
},
|
||
datasource: {
|
||
url: process.env.DATABASE_URL,
|
||
},
|
||
})
|
||
`
|
||
await writeFile(path.join(mainOutDir, 'prisma.config.ts'), localConfigContent, 'utf-8')
|
||
}
|
||
|
||
// Generate package.json
|
||
const optionalDependencies = Object.fromEntries(
|
||
targets.map((t) => [`@shumai-one/${appName}-${t.platform}-${t.arch}`, version]),
|
||
)
|
||
|
||
const dependencies = {
|
||
...dependencyVersions,
|
||
...extraDependencies,
|
||
}
|
||
|
||
const packageJson = {
|
||
name: `@shumai-one/${appName}`,
|
||
version,
|
||
description,
|
||
type: 'module',
|
||
bin: {
|
||
[appName]: `./bin/${appName}.js`,
|
||
},
|
||
files: ['bin', ...(hasPrisma ? ['prisma', 'prisma.config.ts'] : [])],
|
||
dependencies,
|
||
optionalDependencies,
|
||
repository: rootPackageJson.repository,
|
||
}
|
||
|
||
await writeFile(
|
||
path.join(mainOutDir, 'package.json'),
|
||
JSON.stringify(packageJson, null, 2),
|
||
'utf-8',
|
||
)
|
||
}
|
||
|
||
// 2. Build shumai (Main app & platform binaries)
|
||
const shumaiPlugins = [tailwindPlugin]
|
||
const shumaiExternal = commonExternal.filter((dep) => dep !== 'zod')
|
||
await buildPlatformPackages('shumai', dummyRunnerPath, shumaiPlugins, shumaiExternal)
|
||
await buildMainPackage({
|
||
appName: 'shumai',
|
||
description: 'A fullstack AI-powered media workspace and workflow engine',
|
||
hasPrisma: true,
|
||
entrypoint: './apps/web/src/index.ts',
|
||
plugins: shumaiPlugins,
|
||
external: shumaiExternal,
|
||
})
|
||
|
||
// 4. Build shumai-transcode (Transcode app & platform binaries)
|
||
await buildPlatformPackages('shumai-transcode', dummyRunnerPath)
|
||
await buildMainPackage({
|
||
appName: 'shumai-transcode',
|
||
description: 'Media transcoding worker for the shumai media workspace',
|
||
hasPrisma: true,
|
||
entrypoint: './apps/transcode/src/index.ts',
|
||
})
|
||
|
||
// 5. Build shumai-cli (pure Node.js bundle)
|
||
async function buildCliPackage() {
|
||
console.log('\n📄 Packaging CLI app shumai-cli...')
|
||
const cliOutDir = path.join(distDir, 'shumai-cli')
|
||
const binOutDir = path.join(cliOutDir, 'bin')
|
||
await mkdir(binOutDir, { recursive: true })
|
||
|
||
const buildResult = await Bun.build({
|
||
entrypoints: ['./apps/cli/src/index.ts'],
|
||
root: '.',
|
||
target: 'node',
|
||
outdir: binOutDir,
|
||
minify: true,
|
||
naming: 'shumai-cli.js',
|
||
external: [],
|
||
define: {
|
||
'process.env.NODE_ENV': JSON.stringify('production'),
|
||
},
|
||
})
|
||
|
||
if (!buildResult.success) {
|
||
console.error('❌ JS bundling failed for shumai-cli!')
|
||
for (const log of buildResult.logs) {
|
||
console.error(log)
|
||
}
|
||
process.exit(1)
|
||
}
|
||
|
||
// Prepend shebang #!/usr/bin/env node to the output bundle
|
||
const bundlePath = path.join(binOutDir, 'shumai-cli.js')
|
||
const bundleContent = await Bun.file(bundlePath).text()
|
||
await writeFile(bundlePath, `#!/usr/bin/env node\n${bundleContent}`, {
|
||
encoding: 'utf-8',
|
||
mode: 0o755,
|
||
})
|
||
|
||
// Generate package.json for CLI
|
||
const packageJson = {
|
||
name: '@shumai-one/shumai-cli',
|
||
version,
|
||
description: 'Command line interface for the shumai media workspace',
|
||
type: 'module',
|
||
bin: {
|
||
'shumai-cli': './bin/shumai-cli.js',
|
||
},
|
||
files: ['bin'],
|
||
repository: rootPackageJson.repository,
|
||
}
|
||
|
||
await writeFile(
|
||
path.join(cliOutDir, 'package.json'),
|
||
JSON.stringify(packageJson, null, 2),
|
||
'utf-8',
|
||
)
|
||
console.log('✅ CLI package built successfully!')
|
||
}
|
||
|
||
await buildCliPackage()
|
||
|
||
if (existsSync(dummyRunnerPath)) {
|
||
await rm(dummyRunnerPath)
|
||
}
|
||
|
||
console.log('\n✅ All applications and platform binaries built and packaged successfully!')
|