From 97c2939b5e95f9c87b6ecf4c4eae387ba3c7eb96 Mon Sep 17 00:00:00 2001 From: Yiling-J Date: Thu, 12 Nov 2026 19:37:33 +0800 Subject: [PATCH] feat(cli): add standalone downloads and installers (#528) * feat(cli): add standalone downloads and installers * fix(cli): silence PowerShell download progress * fix(cli): use framework APIs for Windows verification * fix(cli): preserve null backup path during Windows updates * docs(cli): translate native installation quickstart into Chinese --- .github/workflows/cli-native.yaml | 49 +++++ .github/workflows/release-npm.yaml | 62 ++++-- CHANGELOG.md | 2 + README.md | 14 ++ README_CN.md | 14 ++ apps/cli/README.md | 73 ++++++- apps/cli/src/program.test.ts | 8 + apps/cli/src/program.ts | 2 + docs/getting-started/quickstart.mdx | 20 ++ docs/zh/getting-started/quickstart.mdx | 20 ++ package.json | 2 + scripts/build-cli.test.ts | 71 ++++++ scripts/build-cli.ts | 128 +++++++++++ scripts/install.ps1 | 88 ++++++++ scripts/install.sh | 95 ++++++++ scripts/install.test.ts | 72 +++++++ scripts/publish-cli-release.test.ts | 150 +++++++++++++ scripts/publish-cli-release.ts | 196 +++++++++++++++++ scripts/publish.ts | 5 +- scripts/test-cli-release.ts | 287 +++++++++++++++++++++++++ 20 files changed, 1339 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/cli-native.yaml create mode 100644 scripts/build-cli.test.ts create mode 100644 scripts/build-cli.ts create mode 100644 scripts/install.ps1 create mode 100644 scripts/install.sh create mode 100644 scripts/install.test.ts create mode 100644 scripts/publish-cli-release.test.ts create mode 100644 scripts/publish-cli-release.ts create mode 100644 scripts/test-cli-release.ts diff --git a/.github/workflows/cli-native.yaml b/.github/workflows/cli-native.yaml new file mode 100644 index 00000000..df176212 --- /dev/null +++ b/.github/workflows/cli-native.yaml @@ -0,0 +1,49 @@ +name: Native CLI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_call: + inputs: + source_ref: + required: true + type: string + +permissions: + contents: read + +jobs: + native: + name: CLI (${{ matrix.target }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { target: linux-x64, runner: ubuntu-22.04 } + - { target: linux-arm64, runner: ubuntu-22.04-arm } + - { target: darwin-x64, runner: macos-15-intel } + - { target: darwin-arm64, runner: macos-15 } + - { target: windows-x64, runner: windows-2022 } + - { target: windows-arm64, runner: windows-11-arm } + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.source_ref || github.ref }} + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.4 + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + - name: Build CLI archive and installer + run: bun run build:cli --target ${{ matrix.target }} + - name: Test installed binary and file transfers + run: bun run test:cli-release + - uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.target }} + path: out/cli-release/shumai-cli-v* + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/release-npm.yaml b/.github/workflows/release-npm.yaml index 61e50c9b..34005317 100644 --- a/.github/workflows/release-npm.yaml +++ b/.github/workflows/release-npm.yaml @@ -14,13 +14,30 @@ on: description: 'Source ref to build/publish (defaults to tag)' required: false type: string + validate_only: + description: 'Build and test without publishing npm packages or a GitHub Release' + required: false + default: false + type: boolean permissions: {} +concurrency: + group: npm-release-${{ github.event.inputs.tag || github.ref_name }} + cancel-in-progress: false + jobs: + cli: + uses: ./.github/workflows/cli-native.yaml + permissions: + contents: read + with: + source_ref: ${{ inputs.source_ref || inputs.tag || github.ref_name }} + release: name: Build, Release and Publish runs-on: ubuntu-latest + needs: cli permissions: contents: write id-token: write @@ -32,6 +49,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.SOURCE_REF }} + fetch-depth: 0 - name: Setup Bun uses: oven-sh/setup-bun@v2 @@ -53,12 +71,18 @@ jobs: - name: Install dependencies run: bun install + - name: Validate release source and version + run: bun scripts/publish-cli-release.ts ${{ inputs.validate_only && 'validate-only' || 'validate' }} + - name: Compile i18n run: bun run i18n:compile - name: Lint run: bun run lint + - name: Format + run: bun run format && git diff --exit-code + - name: Typecheck run: bun run typecheck @@ -68,7 +92,18 @@ jobs: - name: Build NPM packages run: bun run build-npm + - name: Download tested CLI archives + uses: actions/download-artifact@v4 + with: + pattern: cli-* + path: out/cli-release + merge-multiple: true + + - name: Assemble CLI release assets + run: bun run build:cli --assemble + - name: Generate release notes + if: ${{ !inputs.validate_only }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -87,29 +122,24 @@ jobs: fi } > /tmp/release-notes.md - - name: Create GitHub Release + - name: Stage draft GitHub Release and CLI assets + if: ${{ !inputs.validate_only }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if [ ! -s /tmp/release-notes.md ]; then - NOTES_FLAG="--generate-notes" - else - NOTES_FLAG="--notes-file /tmp/release-notes.md" - fi - if gh release view "${{ env.RELEASE_TAG }}" >/dev/null 2>&1; then - gh release edit "${{ env.RELEASE_TAG }}" \ - --title "${{ env.RELEASE_TAG }}" \ - $NOTES_FLAG - else - gh release create "${{ env.RELEASE_TAG }}" \ - --title "${{ env.RELEASE_TAG }}" \ - $NOTES_FLAG - fi + run: bun scripts/publish-cli-release.ts stage - name: Upgrade npm for trusted publishing + if: ${{ !inputs.validate_only }} run: | npm install -g npm@11.16.0 --ignore-scripts npm --version - name: Publish to NPM + if: ${{ !inputs.validate_only }} run: bun run scripts/publish.ts + + - name: Publish completed GitHub Release + if: ${{ !inputs.validate_only }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bun scripts/publish-cli-release.ts publish diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b076071..a9d20660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **cli**: Add standalone downloads and one-line installers for macOS, Linux, and Windows without requiring Node.js or Bun, with verified downloads, version selection, and upgrade support. + ### Fixed - **storage**: Remove empty local folders left behind after generating video search embeddings. diff --git a/README.md b/README.md index e171acdd..1c220ccb 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,20 @@ To set up Shumai locally for development: Shumai provides a Command Line Interface (CLI) tool to manage projects, folders, and assets, upload files/folders, and create new versions directly from your terminal. +Install the standalone CLI on macOS/Linux without Node.js or Bun: + +```sh +curl -fsSL https://github.com/shumaiOne/shumai/releases/latest/download/install.sh | sh +``` + +On Windows: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -Command "irm https://github.com/shumaiOne/shumai/releases/latest/download/install.ps1 | iex" +``` + +The npm installation remains available: `npm install -g @shumai-one/shumai-cli` (Node.js 22.12+). + For more details on installation and usage, see the [CLI Readme](apps/cli/README.md). ## MCP Server diff --git a/README_CN.md b/README_CN.md index 408da90f..4bf6dd0c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -269,6 +269,20 @@ Shumai 提供了独立的转码 Worker 镜像(支持 Docker Hub 与阿里云 Shumai 自带一个功能完备的命令行工具(CLI),支持你直接在终端里高效管理项目、文件夹、素材,批量上传文件/目录,以及快速创建新版本。 +macOS/Linux 可直接安装独立 CLI,无需安装 Node.js 或 Bun: + +```sh +curl -fsSL https://github.com/shumaiOne/shumai/releases/latest/download/install.sh | sh +``` + +Windows 用户可使用 PowerShell: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -Command "irm https://github.com/shumaiOne/shumai/releases/latest/download/install.ps1 | iex" +``` + +仍可通过 npm 安装:`npm install -g @shumai-one/shumai-cli`(需要 Node.js 22.12 或更高版本)。 + 关于 CLI 的安装与详细使用指南,请参阅 [CLI Readme](apps/cli/README.md)。 ## MCP 服务器 diff --git a/apps/cli/README.md b/apps/cli/README.md index 57ca6d22..fbcccd35 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -8,7 +8,63 @@ The CLI allows you to view projects, list folder contents, create folders, uploa ## Installation -The CLI package is bundled inside the Shumai workspace. The published npm package (`@shumai-one/shumai-cli`) requires Node.js 22.12 or later. +### Standalone executable + +Install the CLI without Node.js or Bun. The installers download a platform-specific executable from GitHub Releases and verify its SHA-256 checksum. + +**macOS / Linux:** + +```sh +curl -fsSL https://github.com/shumaiOne/shumai/releases/latest/download/install.sh | sh +``` + +**Windows (PowerShell):** + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -Command "irm https://github.com/shumaiOne/shumai/releases/latest/download/install.ps1 | iex" +``` + +| Platform | Architectures | Requirements | +| --- | --- | --- | +| macOS | Apple Silicon, Intel | macOS 13 or later | +| Linux | ARM64, x64 | glibc 2.17 or later; kernel 5.6+ recommended | +| Windows | ARM64, x64 | Windows 10 version 1809 or later | + +Native builds use Bun 1.4.4's runtime; x64 CPUs need SSE4.2. See [Bun's system requirements](https://bun.com/docs/installation). Alpine/musl is not included in these native downloads. Native downloads are available for releases containing the installer assets; use npm for earlier releases. + +On macOS/Linux the default directory is `~/.local/bin`. Add it to your shell's PATH if needed: + +```sh +export PATH="$HOME/.local/bin:$PATH" +``` + +On Windows the installer uses `%LOCALAPPDATA%\Shumai\bin` and updates the user PATH. Open a new terminal afterward. Administrator access is not required. + +Re-run the installer to upgrade. To pin a version, roll back, or choose an installation directory, download the installer and pass its options (replace `X.Y.Z` with a release containing native binaries): + +```sh +curl -fsSL https://github.com/shumaiOne/shumai/releases/latest/download/install.sh -o install.sh +sh install.sh --version X.Y.Z --install-dir "$HOME/.local/bin" +``` + +```powershell +Invoke-WebRequest https://github.com/shumaiOne/shumai/releases/latest/download/install.ps1 -OutFile install.ps1 +& .\install.ps1 -Version X.Y.Z -InstallDir "$env:LOCALAPPDATA\Shumai\bin" +``` + +The installers preserve the existing executable if download or verification fails and refuse to replace unmanaged installations. If an npm installation takes precedence on PATH, uninstall it with `npm uninstall -g @shumai-one/shumai-cli` or adjust PATH. + +To uninstall a native installation, remove `shumai-cli` (`shumai-cli.exe` on Windows) and `.shumai-cli-install` from its installation directory. Remove the PATH entry if it is no longer needed. + +### npm + +The existing npm package continues to run on Node.js 22.12 or later: + +```sh +npm install -g @shumai-one/shumai-cli +``` + +`shumai-cli --version` reports the installed version for either installation method. ### Environment Setup @@ -31,7 +87,7 @@ export SHUMAI_ALLOW_DELETE="true" ## Usage > [!IMPORTANT] -> **For Bun Users:** Since the package binary contains a `#!/usr/bin/env node` shebang, running `shumai-cli` directly will run under Node.js. If you wish to run it using Bun, prefix the command with `bun run --bun` (e.g., `bun run --bun shumai-cli project ls`). +> **For npm installations using Bun:** The npm entry point uses `#!/usr/bin/env node`. Use `bun run --bun shumai-cli project ls` to run that package under Bun. Standalone installations already include their runtime. Run `shumai-cli` or query specific commands: @@ -258,3 +314,16 @@ bun run apps/cli/src/index.ts project ls ### Adding Commands Most commands are generated from the shared Operation definitions in `packages/operations` (see "Operations" in `AGENTS.md`). An Operation with the `cli` surface and a `cli` spec becomes a command automatically, with arguments, options, and help text derived from its input schema (key/value inputs such as `meta set --values` take a JSON object). It is also available to Shumai's built-in agent and MCP server. Behavior that needs the local machine, such as downloading files, goes in a CLI extension in `src/operations/extensions/` (extra options, input mapping, and output). Only commands that stream local files (`upload`, `create-version`) are written by hand in `src/program.ts`; the `create_file`, `create_version`, and `complete_upload` Operations are for the agent and MCP only. + +### Building and verifying native releases + +```sh +bun run build:cli --target darwin-arm64 # or darwin-x64, linux-arm64, linux-x64, windows-arm64, windows-x64 +bun run test:cli-release +``` + +Without `--target`, the build produces all six targets. Output defaults to `out/cli-release`; both commands accept `--out /absolute/path`. Unix archives require `tar`; creating Windows archives on Unix also requires `zip`. The test runs the generated installer against local HTTP fixtures, then authenticates, creates a folder, uploads and downloads random bytes, and checks failure recovery with no runtime on the executable's PATH. + +The Native CLI workflow builds and tests each target on a matching runner. The npm release workflow assembles those tested archives, stages a draft GitHub Release, publishes npm packages, then publishes the completed release. Its `validate_only` dispatch input builds and tests without publishing. Published assets are verified and preserved on retries; draft assets can be replaced while resuming a failed run. Prerelease npm packages use the `next` tag and prerelease GitHub Releases do not become latest. + +Installers use `SHUMAI_CLI_RELEASE_BASE_URL` only when explicitly set, for private mirrors or local testing. It should point to a release download base whose children are `vX.Y.Z/`. Native executables do not automatically load `.env` or `bunfig.toml`; export the API configuration explicitly as shown above. diff --git a/apps/cli/src/program.test.ts b/apps/cli/src/program.test.ts index 5acce9bf..fe436420 100644 --- a/apps/cli/src/program.test.ts +++ b/apps/cli/src/program.test.ts @@ -25,6 +25,7 @@ import { import type { CallOperation } from './operations/call-operation' import { CliError, formatCliError } from './operations/errors' import { createProgram } from './program' +import { version } from '../package.json' /** Runs the CLI with mocked Operation calls and captures stdout/stderr. */ function setup() { @@ -75,6 +76,13 @@ const pendingRead = { } describe('shumai-cli', () => { + it('reports its package version without calling the server', async () => { + const cli = setup() + await expect(cli.run('--version')).rejects.toMatchObject({ code: 'commander.version' }) + expect(cli.stdout()).toBe(`${version}\n`) + expect(cli.callOperation).not.toHaveBeenCalled() + }) + const originalAllowDelete = process.env.SHUMAI_ALLOW_DELETE afterEach(() => { diff --git a/apps/cli/src/program.ts b/apps/cli/src/program.ts index 81743760..28a11bfa 100644 --- a/apps/cli/src/program.ts +++ b/apps/cli/src/program.ts @@ -6,6 +6,7 @@ import { callOperation, type CallOperation } from './operations/call-operation' import { cliExtensions } from './operations/extensions' import { registerOperationCommands } from './operations/generate' import type { Output } from './operations/render' +import { version } from '../package.json' export interface ProgramOptions { callOperation?: CallOperation @@ -26,6 +27,7 @@ export function createProgram(options: ProgramOptions = {}): Command { const program = new Command() .name('shumai-cli') .description('Command line interface for the Shumai media workspace.') + .version(version) .option('--json', 'Print raw JSON output instead of human-friendly text') .showHelpAfterError() .showSuggestionAfterError() diff --git a/docs/getting-started/quickstart.mdx b/docs/getting-started/quickstart.mdx index 4a55f54d..a80dc3f3 100644 --- a/docs/getting-started/quickstart.mdx +++ b/docs/getting-started/quickstart.mdx @@ -216,3 +216,23 @@ To set up Shumai locally for development: bun run dev ``` This spins up the web server on port `3000` with hot-reloading active. + +--- + +## Install the command-line client + +To connect to a running Shumai server, install the standalone CLI without Node.js or Bun: + +```sh +curl -fsSL https://github.com/shumaiOne/shumai/releases/latest/download/install.sh | sh +``` + +On Windows: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -Command "irm https://github.com/shumaiOne/shumai/releases/latest/download/install.ps1 | iex" +``` + +Native releases support macOS, Linux with glibc, and Windows on ARM64 and x64. The scripts verify SHA-256 checksums and install into your user directory. Re-run the installer to upgrade, and check the installed version with `shumai-cli --version`. The npm alternative, `npm install -g @shumai-one/shumai-cli`, requires Node.js 22.12+. + +Set `SHUMAI_API_SERVER` to your server URL and `SHUMAI_API_KEY` to a personal API token from Settings → API Tokens, then run `shumai-cli project ls`. See the [CLI guide](https://github.com/shumaiOne/shumai/blob/main/apps/cli/README.md) for supported OS versions, PATH setup, pinned versions, rollback, and uninstall instructions. diff --git a/docs/zh/getting-started/quickstart.mdx b/docs/zh/getting-started/quickstart.mdx index 140d0982..3372d3d3 100644 --- a/docs/zh/getting-started/quickstart.mdx +++ b/docs/zh/getting-started/quickstart.mdx @@ -216,3 +216,23 @@ Shumai 提供三种安装方式。对于大多数管理员,推荐使用 **Dock bun run dev ``` 命令执行后将在 `3000` 端口启动 Web 服务,并已开启热重载(Hot-Reloading)。 + +--- + +## 安装命令行客户端 + +若要连接到已运行的 Shumai 服务器,可直接安装独立 CLI,无需安装 Node.js 或 Bun: + +```sh +curl -fsSL https://github.com/shumaiOne/shumai/releases/latest/download/install.sh | sh +``` + +Windows 用户可使用 PowerShell: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -Command "irm https://github.com/shumaiOne/shumai/releases/latest/download/install.ps1 | iex" +``` + +独立版本支持 ARM64 和 x64 架构的 macOS、使用 glibc 的 Linux,以及 Windows。安装脚本会校验 SHA-256 校验和,并将 CLI 安装到当前用户的目录中。重新运行安装脚本即可升级,使用 `shumai-cli --version` 可查看已安装的版本。也可通过 `npm install -g @shumai-one/shumai-cli` 安装,此方式需要 Node.js 22.12 或更高版本。 + +将 `SHUMAI_API_SERVER` 设置为服务器地址,将 `SHUMAI_API_KEY` 设置为在“设置 → API 令牌”中创建的个人 API 令牌,然后运行 `shumai-cli project ls`。有关支持的系统版本、PATH 配置、指定版本安装、回滚及卸载的详细说明,请参阅 [CLI 指南](https://github.com/shumaiOne/shumai/blob/main/apps/cli/README.md)。 diff --git a/package.json b/package.json index cffd0dd7..f8b368ca 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "worker:transcode": "bun apps/transcode/src/index.ts", "build": "bun run i18n:compile && bun run build.ts", "build-npm": "bun run i18n:compile && bun scripts/build-npm.ts", + "build:cli": "bun scripts/build-cli.ts", + "test:cli-release": "bun scripts/test-cli-release.ts", "prepublishOnly": "bun run build-npm && bun run lint && bun run typecheck", "publish": "bun run prepublishOnly && bun run scripts/publish.ts", "publish:dry": "bun run prepublishOnly && bun run scripts/publish.ts --dry-run", diff --git a/scripts/build-cli.test.ts b/scripts/build-cli.test.ts new file mode 100644 index 00000000..861d3649 --- /dev/null +++ b/scripts/build-cli.test.ts @@ -0,0 +1,71 @@ +import { execFileSync } from 'node:child_process' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { version } from '../package.json' +import { + archiveName, + assembleCliRelease, + cliTargetNames, + parseCliTarget, + repoRoot, + sha256, + validateCliVersion, +} from './build-cli' + +describe('CLI release packaging', () => { + it('names all six platform archives and rejects invalid targets and versions', () => { + expect(cliTargetNames).toHaveLength(6) + for (const target of cliTargetNames) { + expect(parseCliTarget(target)).toBe(target) + expect(archiveName(target)).toBe( + `shumai-cli-v${version}-${target}.${target.startsWith('windows-') ? 'zip' : 'tar.gz'}`, + ) + } + for (const value of ['__proto__', 'linux-ia32', '../linux-x64']) + expect(() => parseCliTarget(value)).toThrow() + for (const value of ['', '../1.2.3', '1.2.3;echo bad', 'v1.2.3']) + expect(() => validateCliVersion(value)).toThrow() + expect(validateCliVersion('1.2.3-rc.1')).toBe('1.2.3-rc.1') + }) + + it('checksums the exact release bytes and pins both installers to the package version', async () => { + const directory = await mkdtemp(join(tmpdir(), 'shumai-cli-pack-test-')) + try { + for (const target of cliTargetNames) + await writeFile(join(directory, archiveName(target)), target) + await assembleCliRelease(directory) + const manifest = await readFile(join(directory, 'checksums.txt'), 'utf8') + for (const target of cliTargetNames) + expect(manifest).toContain(`${sha256(target)} ${archiveName(target)}\n`) + for (const name of ['install.sh', 'install.ps1']) { + const script = await readFile(join(directory, name), 'utf8') + expect(script).toContain(version) + expect(script).not.toContain('__SHUMAI_CLI_VERSION__') + } + await rm(join(directory, archiveName('linux-x64'))) + await expect(assembleCliRelease(directory)).rejects.toThrow() + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) + + it.skipIf(process.platform === 'win32')( + 'accepts help and rejects invalid installer arguments before downloading', + () => { + const script = join(repoRoot, 'scripts/install.sh') + expect(execFileSync('/bin/sh', [script, '--help'], { encoding: 'utf8' })).toContain( + '--install-dir', + ) + for (const args of [ + ['--unknown'], + ['--version'], + ['--version', '../1.2.3'], + ['--version', version, '--install-dir', 'relative'], + ]) { + expect(() => execFileSync('/bin/sh', [script, ...args], { stdio: 'pipe' })).toThrow() + } + }, + ) +}) diff --git a/scripts/build-cli.ts b/scripts/build-cli.ts new file mode 100644 index 00000000..fba3341e --- /dev/null +++ b/scripts/build-cli.ts @@ -0,0 +1,128 @@ +import { createHash } from 'node:crypto' +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { spawnSync } from 'node:child_process' +import { join, resolve } from 'node:path' +import { parseArgs } from 'node:util' +import { fileURLToPath } from 'node:url' +import { version } from '../package.json' +import { version as cliVersion } from '../apps/cli/package.json' + +export const cliTargets = { + 'darwin-arm64': 'bun-darwin-arm64', + 'darwin-x64': 'bun-darwin-x64', + 'linux-arm64': 'bun-linux-arm64', + 'linux-x64': 'bun-linux-x64', + 'windows-arm64': 'bun-windows-arm64', + 'windows-x64': 'bun-windows-x64', +} as const +export type CliTarget = keyof typeof cliTargets +export const cliTargetNames = Object.keys(cliTargets) as CliTarget[] +export const repoRoot = fileURLToPath(new URL('..', import.meta.url)) + +export function validateCliVersion(value: string): string { + if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$/.test(value)) { + throw new Error(`Invalid CLI version: ${value}`) + } + return value +} + +export function parseCliTarget(value: string): CliTarget { + if (!Object.hasOwn(cliTargets, value)) throw new Error(`Unsupported CLI target: ${value}`) + return value as CliTarget +} + +export function archiveName(target: CliTarget, releaseVersion = version): string { + validateCliVersion(releaseVersion) + return `shumai-cli-v${releaseVersion}-${target}.${target.startsWith('windows-') ? 'zip' : 'tar.gz'}` +} + +export function sha256(bytes: Uint8Array | string): string { + return createHash('sha256').update(bytes).digest('hex') +} + +export function packCliBinary( + staging: string, + outdir: string, + target: CliTarget, + releaseVersion = version, +) { + const executable = target.startsWith('windows-') ? 'shumai-cli.exe' : 'shumai-cli' + const archive = join(outdir, archiveName(target, releaseVersion)) + const command = target.startsWith('windows-') + ? process.platform === 'win32' + ? ['tar', '-a', '-cf', archive, '-C', staging, executable] + : ['zip', '-j', archive, join(staging, executable)] + : ['tar', '-czf', archive, '-C', staging, executable] + const packed = spawnSync(command[0], command.slice(1), { stdio: 'inherit' }) + if (packed.error) throw packed.error + if (packed.status !== 0) throw new Error(`Failed to archive ${target}`) + return archive +} + +export async function assembleCliRelease(outdir: string, targets = cliTargetNames) { + validateCliVersion(version) + if (cliVersion !== version) throw new Error('Root and CLI package versions must match') + for (const name of ['install.sh', 'install.ps1']) { + const template = await readFile(join(repoRoot, 'scripts', name), 'utf8') + await writeFile(join(outdir, name), template.replaceAll('__SHUMAI_CLI_VERSION__', version)) + } + const names = [ + ...targets.map((target) => archiveName(target)), + 'install.sh', + 'install.ps1', + ].sort() + const lines = await Promise.all( + names.map(async (name) => `${sha256(await readFile(join(outdir, name)))} ${name}\n`), + ) + await writeFile(join(outdir, 'checksums.txt'), lines.join('')) +} + +export async function buildCliRelease(outdir: string, targets = cliTargetNames) { + validateCliVersion(version) + if (cliVersion !== version) throw new Error('Root and CLI package versions must match') + await mkdir(outdir, { recursive: true }) + const staging = await mkdtemp(join(outdir, '.build-')) + try { + for (const target of targets) { + const executable = target.startsWith('windows-') ? 'shumai-cli.exe' : 'shumai-cli' + const binary = join(staging, executable) + const result = await Bun.build({ + entrypoints: [join(repoRoot, 'apps/cli/src/index.ts')], + target: 'bun', + minify: true, + compile: { + target: cliTargets[target], + outfile: binary, + autoloadDotenv: false, + autoloadBunfig: false, + autoloadPackageJson: false, + autoloadTsconfig: false, + }, + define: { 'process.env.NODE_ENV': JSON.stringify('production') }, + }) + if (!result.success) throw new AggregateError(result.logs, `Failed to compile ${target}`) + await chmod(binary, 0o755) + const archive = join(outdir, archiveName(target)) + await rm(archive, { force: true }) + packCliBinary(staging, outdir, target) + console.log(`Built ${archive}`) + } + await assembleCliRelease(outdir, targets) + } finally { + await rm(staging, { recursive: true, force: true }) + } +} + +if (import.meta.main) { + const { values } = parseArgs({ + options: { + target: { type: 'string' }, + out: { type: 'string', default: 'out/cli-release' }, + assemble: { type: 'boolean', default: false }, + }, + }) + const targets = values.target ? [parseCliTarget(values.target)] : cliTargetNames + const outdir = resolve(values.out) + if (values.assemble) await assembleCliRelease(outdir, targets) + else await buildCliRelease(outdir, targets) +} diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 00000000..9f32ed35 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,88 @@ +param( + [string]$Version = '__SHUMAI_CLI_VERSION__', + [string]$InstallDir = (Join-Path $env:LOCALAPPDATA 'Shumai\bin'), + [switch]$NoModifyPath +) +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$Version = $Version -replace '^v', '' +if ($Version -notmatch '^\d+\.\d+\.\d+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$') { + throw 'Use a release installer or pass -Version X.Y.Z' +} +if (-not [IO.Path]::IsPathRooted($InstallDir)) { throw '-InstallDir must be an absolute path' } +if ([Environment]::OSVersion.Platform -ne [PlatformID]::Win32NT) { throw 'Use install.sh on macOS or Linux' } +# Read the native OS architecture from the machine environment, avoiding both +# process emulation and PSReadLine's conflicting RuntimeInformation type on PS 5.1. +$architecture = [Environment]::GetEnvironmentVariable('PROCESSOR_ARCHITECTURE', 'Machine') +if (!$architecture) { $architecture = $env:PROCESSOR_ARCHITEW6432 } +if (!$architecture) { $architecture = $env:PROCESSOR_ARCHITECTURE } +$architecture = "$architecture".ToLowerInvariant() +switch ($architecture) { + 'arm64' { $arch = 'arm64' } + 'amd64' { $arch = 'x64' } + 'x64' { $arch = 'x64' } + default { throw "Unsupported CPU architecture: $architecture" } +} +$destination = Join-Path $InstallDir 'shumai-cli.exe' +$marker = Join-Path $InstallDir '.shumai-cli-install' +if (Test-Path -LiteralPath $destination) { + if ((Get-Item -LiteralPath $destination).Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'Refusing to replace a linked installation' } + if (!(Test-Path -LiteralPath $marker) -or (Get-Content -LiteralPath $marker -Raw).Trim() -ne 'shumai-cli') { + throw "Refusing to replace an unmanaged installation at $destination; choose -InstallDir" + } +} +$temporary = Join-Path ([IO.Path]::GetTempPath()) ([IO.Path]::GetRandomFileName()) +$staged = $null +try { + $null = New-Item -ItemType Directory -Path $temporary + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $base = 'https://github.com/shumaiOne/shumai/releases/download' + if ($env:SHUMAI_CLI_RELEASE_BASE_URL) { $base = $env:SHUMAI_CLI_RELEASE_BASE_URL.TrimEnd('/') } + $base = "$base/v$Version" + $archive = "shumai-cli-v$Version-windows-$arch.zip" + $manifest = Join-Path $temporary 'checksums.txt' + Invoke-WebRequest -UseBasicParsing -Uri "$base/checksums.txt" -OutFile $manifest + $pattern = '^[a-f0-9]{64} ' + [regex]::Escape($archive) + '$' + $matchesChecksum = @(Get-Content -LiteralPath $manifest | Where-Object { $_ -cmatch $pattern }) + if ($matchesChecksum.Count -ne 1) { throw "Missing or ambiguous checksum for $archive" } + $archivePath = Join-Path $temporary $archive + Invoke-WebRequest -UseBasicParsing -Uri "$base/$archive" -OutFile $archivePath + # Use framework APIs so installation does not depend on module discovery + # inherited from a parent PowerShell session of a different version. + $hasher = [Security.Cryptography.SHA256]::Create() + try { + $stream = [IO.File]::OpenRead($archivePath) + try { $actual = [BitConverter]::ToString($hasher.ComputeHash($stream)).Replace('-', '').ToLowerInvariant() } + finally { $stream.Dispose() } + } + finally { $hasher.Dispose() } + if ($actual -cne $matchesChecksum[0].Substring(0, 64)) { throw 'Download checksum does not match; existing installation preserved' } + Add-Type -AssemblyName System.IO.Compression.FileSystem + [IO.Compression.ZipFile]::ExtractToDirectory($archivePath, $temporary) + $executable = Join-Path $temporary 'shumai-cli.exe' + if (!(Test-Path -LiteralPath $executable -PathType Leaf)) { throw 'Archive does not contain the CLI executable' } + $binaryVersion = & $executable --version + if ($LASTEXITCODE -ne 0 -or "$binaryVersion".Trim() -ne $Version) { throw 'Downloaded CLI version does not match' } + $null = New-Item -ItemType Directory -Path $InstallDir -Force + $staged = Join-Path $InstallDir ([IO.Path]::GetRandomFileName()) + Copy-Item -LiteralPath $executable -Destination $staged + Set-Content -LiteralPath $marker -Value 'shumai-cli' -Encoding ASCII + # PowerShell 5.1 converts $null to an empty string for string parameters. + if (Test-Path -LiteralPath $destination) { [IO.File]::Replace($staged, $destination, [NullString]::Value) } + else { [IO.File]::Move($staged, $destination) } + $staged = $null + if (!$NoModifyPath) { + $userPath = [Environment]::GetEnvironmentVariable('Path', 'User') + if (($userPath -split ';') -notcontains $InstallDir) { + [Environment]::SetEnvironmentVariable('Path', "$InstallDir;$userPath", 'User') + } + Write-Output 'Open a new terminal to use the updated user PATH.' + } + Write-Output "Installed shumai-cli $Version to $destination" + $resolved = Get-Command shumai-cli -ErrorAction SilentlyContinue + if ($resolved -and $resolved.Source -ne $destination) { Write-Output "Your PATH currently selects another installation: $($resolved.Source)" } +} +finally { + if ($staged -and (Test-Path -LiteralPath $staged)) { Remove-Item -LiteralPath $staged -Force } + if (Test-Path -LiteralPath $temporary) { Remove-Item -LiteralPath $temporary -Recurse -Force } +} diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 00000000..21d833ab --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# Release builds replace this placeholder with the version of the attached binaries. +set -eu + +fail() { printf 'Error: %s\n' "$*" >&2; exit 1; } + +main() { + version='__SHUMAI_CLI_VERSION__' + install_dir="${HOME:?HOME must be set}/.local/bin" + while [ "$#" -gt 0 ]; do + case "$1" in + --version|--install-dir) + [ "$#" -ge 2 ] && [ -n "$2" ] || fail "$1 requires a value" + case "$1" in --version) version=${2#v} ;; --install-dir) install_dir=$2 ;; esac + shift 2 ;; + --help|-h) + printf 'Usage: install.sh [--version X.Y.Z] [--install-dir DIRECTORY]\n' + return ;; + *) fail "Unknown option: $1" ;; + esac + done + printf '%s\n' "$version" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$' || fail 'Use a release installer or pass --version X.Y.Z' + case "$install_dir" in /*) ;; *) fail '--install-dir must be an absolute path' ;; esac + + case "$(uname -s)" in + Darwin) platform=darwin ;; + Linux) + platform=linux + if ldd --version 2>&1 | grep -qi musl; then fail 'Linux musl/Alpine is not supported; use the npm package instead'; fi ;; + *) fail 'Supported systems are macOS and Linux with glibc' ;; + esac + machine=$(uname -m) + if [ "$platform" = darwin ] && [ "$(sysctl -n hw.optional.arm64 2>/dev/null || true)" = 1 ]; then machine=arm64; fi + case "$machine" in + arm64|aarch64) arch=arm64 ;; + x86_64|amd64) arch=x64 ;; + *) fail "Unsupported CPU architecture: $machine" ;; + esac + + command -v tar >/dev/null 2>&1 || fail 'tar is required' + if command -v curl >/dev/null 2>&1; then downloader=curl + elif command -v wget >/dev/null 2>&1; then downloader=wget + else fail 'curl or wget is required'; fi + if command -v sha256sum >/dev/null 2>&1; then hasher=sha256sum + elif command -v shasum >/dev/null 2>&1; then hasher=shasum + else fail 'sha256sum or shasum is required'; fi + + destination="$install_dir/shumai-cli" + marker="$install_dir/.shumai-cli-install" + if [ -e "$destination" ] || [ -L "$destination" ]; then + [ ! -L "$destination" ] && [ -f "$marker" ] && [ "$(cat "$marker")" = shumai-cli ] || fail "Refusing to replace an unmanaged installation at $destination; choose --install-dir" + fi + + temporary=$(mktemp -d "${TMPDIR:-/tmp}/shumai-install.XXXXXX") + staged='' + trap 'rm -rf "$temporary"; if [ -n "$staged" ]; then rm -f "$staged"; fi' 0 + trap 'exit 1' HUP INT TERM + archive="shumai-cli-v$version-$platform-$arch.tar.gz" + base="${SHUMAI_CLI_RELEASE_BASE_URL:-https://github.com/shumaiOne/shumai/releases/download}/v$version" + download() { + if [ "$downloader" = curl ]; then curl -fsSL --retry 2 --connect-timeout 10 "$base/$1" -o "$2" + else wget -q "$base/$1" -O "$2"; fi + } + download checksums.txt "$temporary/checksums.txt" + expected=$(awk -v name="$archive" '$2 == name { print $1 }' "$temporary/checksums.txt") + [ "${#expected}" -eq 64 ] || fail "Missing or ambiguous checksum for $archive" + case "$expected" in *[!0-9a-f]*) fail 'Invalid SHA-256 checksum' ;; esac + download "$archive" "$temporary/$archive" + if [ "$hasher" = sha256sum ]; then actual=$(sha256sum "$temporary/$archive") + else actual=$(shasum -a 256 "$temporary/$archive"); fi + [ "${actual%% *}" = "$expected" ] || fail 'Download checksum does not match; existing installation preserved' + + tar -xzf "$temporary/$archive" -C "$temporary" shumai-cli + [ -f "$temporary/shumai-cli" ] && [ ! -L "$temporary/shumai-cli" ] || fail 'Archive does not contain the CLI executable' + chmod 755 "$temporary/shumai-cli" + [ "$("$temporary/shumai-cli" --version)" = "$version" ] || fail 'Downloaded CLI version does not match' + mkdir -p "$install_dir" + staged=$(mktemp "$install_dir/.shumai-cli.XXXXXX") + cp "$temporary/shumai-cli" "$staged" + chmod 755 "$staged" + printf 'shumai-cli\n' > "$marker" + mv -f "$staged" "$destination" + staged='' + printf 'Installed shumai-cli %s to %s\n' "$version" "$destination" + case ":${PATH:-}:" in + *":$install_dir:"*) ;; + *) printf 'Add this directory to PATH in your shell configuration: %s\n' "$install_dir" ;; + esac + resolved=$(command -v shumai-cli || true) + if [ -n "$resolved" ] && [ "$resolved" != "$destination" ]; then + printf 'Your PATH currently selects another installation: %s\n' "$resolved" + fi +} + +main "$@" diff --git a/scripts/install.test.ts b/scripts/install.test.ts new file mode 100644 index 00000000..4aca850f --- /dev/null +++ b/scripts/install.test.ts @@ -0,0 +1,72 @@ +import { execFileSync } from 'node:child_process' +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { version } from '../package.json' +import { repoRoot } from './build-cli' + +describe.skipIf(process.platform === 'win32')('Unix installer preflight', () => { + let temporary: string + let bin: string + const command = async (name: string, body: string) => { + const file = join(bin, name) + await writeFile(file, `#!/bin/sh\n${body}\n`) + await chmod(file, 0o755) + } + const install = () => { + try { + execFileSync( + '/bin/sh', + [ + join(repoRoot, 'scripts/install.sh'), + '--version', + version, + '--install-dir', + join(temporary, 'install'), + ], + { + env: { PATH: bin, HOME: temporary }, + stdio: 'pipe', + }, + ) + throw new Error('Installer unexpectedly succeeded') + } catch (error) { + if (error && typeof error === 'object' && 'stderr' in error && Buffer.isBuffer(error.stderr)) + return error.stderr.toString() + throw error + } + } + beforeEach(async () => { + temporary = await mkdtemp(join(tmpdir(), 'shumai-installer-test-')) + bin = join(temporary, 'bin') + await mkdir(bin) + await symlink('/usr/bin/grep', join(bin, 'grep')) + await command('uname', 'case "$1" in -s) printf Darwin ;; -m) printf x86_64 ;; esac') + await command('sysctl', 'printf 0') + }) + afterEach(async () => { + await rm(temporary, { recursive: true, force: true }) + }) + + it('rejects unsupported operating systems and CPU architectures', async () => { + await command('uname', 'printf FreeBSD') + expect(install()).toContain('Supported systems') + await command('uname', 'case "$1" in -s) printf Darwin ;; -m) printf riscv64 ;; esac') + expect(install()).toContain('Unsupported CPU architecture') + }) + + it('rejects musl instead of downloading a glibc executable', async () => { + await command('uname', 'case "$1" in -s) printf Linux ;; -m) printf x86_64 ;; esac') + await command('ldd', 'printf musl') + expect(install()).toContain('musl/Alpine is not supported') + }) + + it('reports missing extraction, download, and hashing tools before downloading', async () => { + expect(install()).toContain('tar is required') + await command('tar', 'exit 0') + expect(install()).toContain('curl or wget is required') + await command('curl', 'exit 99') + expect(install()).toContain('sha256sum or shasum is required') + }) +}) diff --git a/scripts/publish-cli-release.test.ts b/scripts/publish-cli-release.test.ts new file mode 100644 index 00000000..172795c7 --- /dev/null +++ b/scripts/publish-cli-release.test.ts @@ -0,0 +1,150 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { version } from '../package.json' +import { archiveName, assembleCliRelease, cliTargetNames, sha256 } from './build-cli' +import { + cliReleaseFiles, + parseChecksums, + publishCliRelease, + validateReleaseSource, +} from './publish-cli-release' + +describe('CLI release publication', () => { + let directory: string + let manifest: string + let assets: { name: string; digest: string }[] + let draft: boolean | undefined + let source = 'release-commit' + const calls: string[][] = [] + const run = vi.fn((command: string, args: string[]): string => { + calls.push([command, ...args]) + if (command === 'git') return args[1] === 'HEAD' ? source : 'release-commit' + if (args[0] === 'api') { + if (draft === undefined) throw new Error('gh: Not Found (HTTP 404)') + return JSON.stringify({ draft, assets }) + } + if (args[1] === 'create') draft = true + if (args[1] === 'download') return manifest + if (args.includes('--draft=false')) draft = false + return '' + }) + const context = () => ({ + tag: `v${version}`, + repository: 'shumaiOne/shumai', + outdir: directory, + notes: join(directory, 'notes.md'), + }) + + beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'shumai-publish-test-')) + for (const target of cliTargetNames) + await writeFile(join(directory, archiveName(target)), target) + await assembleCliRelease(directory) + manifest = await readFile(join(directory, 'checksums.txt'), 'utf8') + assets = await Promise.all( + cliReleaseFiles().map(async (name) => ({ + name, + digest: `sha256:${sha256(await readFile(join(directory, name)))}`, + })), + ) + await writeFile(join(directory, 'notes.md'), 'Test release notes') + draft = undefined + source = 'release-commit' + calls.length = 0 + run.mockClear() + }) + + afterEach(async () => { + await rm(directory, { recursive: true, force: true }) + }) + + it('stages all assets in a draft, can resume staging, and only publishes in the final step', async () => { + await publishCliRelease('stage', context(), run) + expect(draft).toBe(true) + expect(calls.some((call) => call.includes('--verify-tag') && call.includes('--draft'))).toBe( + true, + ) + const upload = calls.find((call) => call[2] === 'upload')! + for (const name of cliReleaseFiles()) expect(upload).toContain(join(directory, name)) + expect(upload).toContain('--clobber') + await publishCliRelease('stage', context(), run) + expect(calls.filter((call) => call[2] === 'create')).toHaveLength(1) + expect(draft).toBe(true) + await publishCliRelease('publish', context(), run) + expect(draft).toBe(false) + expect(calls.at(-1)).toContain('--latest=true') + }) + + it('validates published artifacts without replacing them, even when a rebuild differs', async () => { + draft = false + await writeFile(join(directory, archiveName('linux-x64')), 'different archive timestamp') + await assembleCliRelease(directory) + await publishCliRelease('stage', context(), run) + await publishCliRelease('publish', context(), run) + expect( + calls + .filter((call) => call[0] === 'gh') + .every((call) => call[1] === 'api' || call[2] === 'download'), + ).toBe(true) + }) + + it('rejects a source/tag mismatch before making any GitHub changes', async () => { + source = 'another-commit' + await expect(publishCliRelease('stage', context(), run)).rejects.toThrow('Source ref') + expect(calls.every((call) => call[0] === 'git')).toBe(true) + expect(() => validateReleaseSource('v99.0.0', run)).toThrow('package version') + expect(() => validateReleaseSource(`v${version}`, run, true)).not.toThrow() + }) + + it('rejects tampered local archives before accessing GitHub', async () => { + await writeFile(join(directory, archiveName('linux-x64')), 'tampered') + await expect(publishCliRelease('stage', context(), run)).rejects.toThrow( + 'Local checksum mismatch', + ) + expect(calls.every((call) => call[0] === 'git')).toBe(true) + }) + + it.each([true, false])( + 'does not publish or modify incomplete releases (draft=%s)', + async (isDraft) => { + draft = isDraft + assets = assets.filter((asset) => asset.name !== 'install.ps1') + await expect(publishCliRelease('publish', context(), run)).rejects.toThrow( + 'Missing or invalid GitHub release asset', + ) + expect(calls.some((call) => call.includes('--draft=false') || call[2] === 'upload')).toBe( + false, + ) + }, + ) + + it('rejects mismatched remote digests and duplicate or missing checksum entries', async () => { + draft = true + assets[0].digest = `sha256:${'0'.repeat(64)}` + await expect(publishCliRelease('publish', context(), run)).rejects.toThrow( + 'Missing or invalid GitHub release asset', + ) + const names = cliReleaseFiles().filter((name) => name !== 'checksums.txt') + expect(() => parseChecksums(manifest + manifest, names)).toThrow('Duplicate') + expect(() => parseChecksums(manifest.split('\n').slice(1).join('\n'), names)).toThrow( + 'incomplete', + ) + expect(() => parseChecksums('bad manifest', names)).toThrow('Invalid') + }) + + it('does not interpret network errors as missing releases or create an unstaged release', async () => { + run + .mockImplementationOnce(() => 'release-commit') + .mockImplementationOnce(() => 'release-commit') + .mockImplementationOnce(() => { + throw new Error('Network unavailable') + }) + await expect(publishCliRelease('stage', context(), run)).rejects.toThrow('Network unavailable') + expect(calls.some((call) => call[2] === 'create')).toBe(false) + await expect(publishCliRelease('publish', context(), run)).rejects.toThrow( + 'staged before publishing', + ) + }) +}) diff --git a/scripts/publish-cli-release.ts b/scripts/publish-cli-release.ts new file mode 100644 index 00000000..18586494 --- /dev/null +++ b/scripts/publish-cli-release.ts @@ -0,0 +1,196 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { version } from '../package.json' +import { archiveName, cliTargetNames, sha256, validateCliVersion } from './build-cli' + +type Runner = (command: string, args: string[]) => string +interface ReleaseAsset { + name: string + digest: string | null +} +interface Release { + isDraft: boolean + assets: ReleaseAsset[] +} + +export function cliReleaseFiles() { + return [ + ...cliTargetNames.map((target) => archiveName(target)), + 'install.sh', + 'install.ps1', + 'checksums.txt', + ] +} + +export function parseChecksums(text: string, names: string[]): Map { + const hashes = new Map() + for (const line of text.trim().split(/\r?\n/)) { + const match = /^([a-f0-9]{64}) {2}([a-zA-Z0-9.-]+)$/.exec(line) + assert.ok(match && names.includes(match[2]), 'Invalid release checksum entry') + assert.ok(!hashes.has(match[2]), 'Duplicate release checksum entry') + hashes.set(match[2], match[1]) + } + assert.equal(hashes.size, names.length, 'Release checksum entries are incomplete') + return hashes +} + +export function validateReleaseSource(tag: string, run: Runner, validationOnly = false) { + validateCliVersion(version) + assert.equal(tag, `v${version}`, 'Release tag must match the package version') + if (validationOnly) return + assert.equal( + run('git', ['rev-parse', 'HEAD']).trim(), + run('git', ['rev-parse', `${tag}^{commit}`]).trim(), + 'Source ref must point to the release tag commit', + ) +} + +function parseRelease(value: unknown): Release { + assert.ok( + value && typeof value === 'object' && 'draft' in value && typeof value.draft === 'boolean', + ) + assert.ok('assets' in value && Array.isArray(value.assets)) + const assets: unknown[] = value.assets + return { + isDraft: value.draft, + assets: assets.map((asset) => { + assert.ok( + asset && typeof asset === 'object' && 'name' in asset && typeof asset.name === 'string', + ) + const digest = 'digest' in asset && typeof asset.digest === 'string' ? asset.digest : null + return { name: asset.name, digest } + }), + } +} + +export async function publishCliRelease( + mode: 'stage' | 'publish', + context: { tag: string; repository: string; outdir: string; notes: string }, + run: Runner, +) { + validateReleaseSource(context.tag, run) + assert.match(context.repository, /^[\w.-]+\/[\w.-]+$/) + const names = cliReleaseFiles() + const checksummed = names.filter((name) => name !== 'checksums.txt') + const localHashes = parseChecksums( + await readFile(join(context.outdir, 'checksums.txt'), 'utf8'), + checksummed, + ) + for (const name of checksummed) + assert.equal( + sha256(await readFile(join(context.outdir, name))), + localHashes.get(name), + `Local checksum mismatch: ${name}`, + ) + + const endpoint = `repos/${context.repository}/releases/tags/${context.tag}` + let release: Release | undefined + try { + release = parseRelease(JSON.parse(run('gh', ['api', endpoint]))) + } catch (error) { + if (!(error instanceof Error) || !error.message.includes('HTTP 404')) throw error + } + if (!release) { + assert.equal(mode, 'stage', 'Release must be staged before publishing') + run('gh', [ + 'release', + 'create', + context.tag, + '--verify-tag', + '--draft', + '--title', + context.tag, + '--notes-file', + context.notes, + ]) + release = { isDraft: true, assets: [] } + } + + if (mode === 'stage' && release.isDraft) { + run('gh', [ + 'release', + 'edit', + context.tag, + '--title', + context.tag, + '--notes-file', + context.notes, + ]) + // Only draft assets may be replaced when resuming a failed build/publication. + run('gh', [ + 'release', + 'upload', + context.tag, + ...names.map((name) => join(context.outdir, name)), + '--clobber', + ]) + return + } + + // A successful rerun validates the existing published bytes instead of replacing them + // with a rebuild whose archive timestamps can differ. + const manifest = run('gh', [ + 'release', + 'download', + context.tag, + '--pattern', + 'checksums.txt', + '--output', + '-', + ]) + const remoteHashes = parseChecksums(manifest, checksummed) + remoteHashes.set('checksums.txt', sha256(manifest)) + for (const name of names) { + const asset: ReleaseAsset | undefined = release.assets.find((entry) => entry.name === name) + assert.equal( + asset?.digest, + `sha256:${remoteHashes.get(name)}`, + `Missing or invalid GitHub release asset: ${name}`, + ) + } + if (mode === 'publish' && release.isDraft) { + const prerelease = version.includes('-') + run('gh', [ + 'release', + 'edit', + context.tag, + '--draft=false', + `--prerelease=${prerelease}`, + `--latest=${!prerelease}`, + ]) + } +} + +if (import.meta.main) { + const run: Runner = (command, args) => { + const result = spawnSync(command, args, { + encoding: 'utf8', + stdio: ['inherit', 'pipe', 'pipe'], + }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`${command} ${args.join(' ')}: ${result.stderr}`) + return result.stdout + } + const tag = process.env.RELEASE_TAG ?? '' + const mode = process.argv[2] + if (mode === 'validate' || mode === 'validate-only') + validateReleaseSource(tag, run, mode === 'validate-only') + else { + assert.ok( + mode === 'stage' || mode === 'publish', + 'Usage: publish-cli-release.ts ', + ) + await publishCliRelease( + mode, + { + tag, + repository: process.env.GITHUB_REPOSITORY ?? 'shumaiOne/shumai', + outdir: 'out/cli-release', + notes: '/tmp/release-notes.md', + }, + run, + ) + } +} diff --git a/scripts/publish.ts b/scripts/publish.ts index ef4c2c39..3d5cdc07 100644 --- a/scripts/publish.ts +++ b/scripts/publish.ts @@ -142,6 +142,9 @@ for (const pkg of packages) { continue } - run('npm', ['publish', '--access', 'public', '--ignore-scripts'], { cwd: pkg.directory }) + const npmDistTag = version.includes('-') ? 'next' : 'latest' + run('npm', ['publish', '--access', 'public', '--ignore-scripts', '--tag', npmDistTag], { + cwd: pkg.directory, + }) console.log() } diff --git a/scripts/test-cli-release.ts b/scripts/test-cli-release.ts new file mode 100644 index 00000000..07461aac --- /dev/null +++ b/scripts/test-cli-release.ts @@ -0,0 +1,287 @@ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { promisify, parseArgs } from 'node:util' +import { randomBytes } from 'node:crypto' +import { version } from '../package.json' +import { archiveName, packCliBinary, parseCliTarget, sha256 } from './build-cli' + +const execFileAsync = promisify(execFile) + +function assertRecord(value: unknown): asserts value is Record { + assert.ok(value && typeof value === 'object' && !Array.isArray(value)) +} + +/** Exercises the packaged installer and executable against HTTP, without a runtime on PATH. */ +export async function testCliRelease(outdir: string) { + const windows = process.platform === 'win32' + const target = parseCliTarget(`${windows ? 'windows' : process.platform}-${process.arch}`) + const bytes = randomBytes(65536) + const asset = { + id: 'file-test', + name: 'random.bin', + type: 'file', + size: bytes.length, + projectId: 'project-test', + } + let transferred = false + let confirmed = false + let mode: 'normal' | 'corrupt' | 'missing' | 'ambiguous' = 'normal' + let fixtureFailure: unknown + const olderVersion = '0.0.1' + let olderDirectory = '' + const server = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + async fetch(request) { + try { + const url = new URL(request.url) + if (url.pathname.startsWith('/releases/')) { + const requestedVersion = url.pathname.split('/')[2]?.slice(1) + assert.ok( + requestedVersion === version || requestedVersion === olderVersion, + 'Installer must pin the version', + ) + if (mode === 'missing') return new Response('Missing asset', { status: 404 }) + const name = url.pathname.split('/').at(-1)! + if (name === 'checksums.txt' && mode === 'ambiguous') { + const manifest = await readFile(join(outdir, name)) + return new Response(Buffer.concat([manifest, manifest])) + } + if (name === archiveName(target) && mode === 'corrupt') + return new Response('Corrupted archive') + assert.ok(name === 'checksums.txt' || name === archiveName(target, requestedVersion)) + return new Response( + Bun.file(join(requestedVersion === olderVersion ? olderDirectory : outdir, name)), + ) + } + if (url.pathname === '/upload' && request.method === 'PUT') { + assert.deepEqual(Buffer.from(await request.arrayBuffer()), bytes) + assert.equal(Number(request.headers.get('content-length')), bytes.length) + transferred = true + return new Response(null, { status: 200 }) + } + if (url.pathname === '/original') return new Response(bytes) + if (request.headers.get('authorization') !== 'Bearer fixture-token') { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + if (url.pathname.startsWith('/api/ops/')) { + assert.equal(request.method, 'POST') + const input: unknown = await request.json() + assertRecord(input) + switch (url.pathname.slice('/api/ops/'.length)) { + case 'list_projects': + return Response.json({ + data: [ + { + id: 'project-test', + name: 'Native CLI', + teamId: 'team-test', + rootFolderId: 'root-test', + }, + ], + }) + case 'create_folder': + assert.deepEqual(input, { parent: 'root-test', name: 'Uploads' }) + return Response.json({ + id: 'folder-test', + name: 'Uploads', + type: 'folder', + size: 0, + projectId: 'project-test', + }) + case 'list_assets': + assert.equal(input.parent, 'folder-test') + assert.ok(confirmed) + return Response.json({ + data: [asset], + total: 1, + pageInfo: { hasNextPage: false, endCursor: null }, + }) + case 'download_asset': + assert.deepEqual(input, { assetIds: ['file-test'], proxy: false }) + return Response.json({ + files: [ + { + assetId: asset.id, + name: asset.name, + kind: 'original', + size: bytes.length, + mimeType: 'application/octet-stream', + url: `${url.origin}/original`, + }, + ], + }) + } + } + if (url.pathname === '/api/folders/folder-test') + return Response.json({ projectId: 'project-test' }) + if (url.pathname === '/api/projects/project-test') + return Response.json({ teamId: 'team-test' }) + if (url.pathname === '/api/teams/team-test/upload/tasks' && request.method === 'POST') { + const input: unknown = await request.json() + assertRecord(input) + assert.equal(input.parentId, 'folder-test') + assert.ok(Array.isArray(input.files)) + const files: unknown[] = input.files + const file = files[0] + assertRecord(file) + assert.equal(file.name, asset.name) + assert.equal(file.size, bytes.length) + return Response.json({ + taskId: 'task-test', + presignedUrls: [{ id: file.id, fileId: asset.id, url: `${url.origin}/upload` }], + createdAssets: [{ id: asset.id, tempId: file.id }], + }) + } + if ( + url.pathname === '/api/teams/team-test/upload/tasks/task-test' && + request.method === 'PATCH' + ) { + assert.ok(transferred) + assert.deepEqual(await request.json(), { fileId: asset.id }) + confirmed = true + return Response.json({ success: true }) + } + throw new Error(`Unexpected request: ${request.method} ${url.pathname}`) + } catch (error) { + fixtureFailure ??= error + return new Response('Fixture assertion failed', { status: 500 }) + } + }, + }) + const temporary = await mkdtemp(join(tmpdir(), 'shumai-cli-release-')) + const installDir = join(temporary, 'installed cli') + const binary = join(installDir, windows ? 'shumai-cli.exe' : 'shumai-cli') + const installerEnv = { + ...process.env, + SHUMAI_CLI_RELEASE_BASE_URL: `${server.url.origin}/releases`, + } + const install = (selectedVersion?: string) => { + const psQuote = (value: string) => `'${value.replaceAll("'", "''")}'` + // PSReadLine can shadow RuntimeInformation in Windows PowerShell 5.1. + const psCommand = `$ErrorActionPreference = 'Stop'; Import-Module PSReadLine; & ${psQuote(join(outdir, 'install.ps1'))} -InstallDir ${psQuote(installDir)} -NoModifyPath ${selectedVersion ? `-Version ${psQuote(selectedVersion)}` : ''}` + return execFileAsync( + windows ? 'powershell.exe' : '/bin/sh', + windows + ? [ + '-NoProfile', + '-ExecutionPolicy', + 'Bypass', + '-EncodedCommand', + Buffer.from(psCommand, 'utf16le').toString('base64'), + ] + : [ + join(outdir, 'install.sh'), + '--install-dir', + installDir, + ...(selectedVersion ? ['--version', selectedVersion] : []), + ], + { env: installerEnv, timeout: 60000 }, + ) + } + const cliEnv: NodeJS.ProcessEnv = { + ...process.env, + PATH: '', + HOME: temporary, + SHUMAI_API_SERVER: server.url.origin, + SHUMAI_API_KEY: 'fixture-token', + } + delete cliEnv.BUN_BE_BUN + delete cliEnv.BUN_OPTIONS + const run = async (args: string[], env = cliEnv) => { + const result = await execFileAsync(binary, args, { cwd: temporary, env, timeout: 30000 }) + if (fixtureFailure) throw fixtureFailure + return result.stdout + } + try { + await install() + assert.equal((await run(['--version'])).trim(), version) + assert.match(await run(['--help']), /Command line interface/) + const projects = JSON.parse(await run(['--json', 'project', 'ls'])) as { + data: { id: string }[] + } + assert.equal(projects.data[0].id, 'project-test') + const folder = JSON.parse(await run(['--json', 'mkdir', 'Uploads', '-p', 'root-test'])) as { + id: string + } + assert.equal(folder.id, 'folder-test') + const file = join(temporary, asset.name) + await writeFile(file, bytes) + await run(['upload', file, '-p', folder.id]) + assert.ok(confirmed, 'Upload must transfer and confirm the bytes, not merely exit successfully') + assert.match(await run(['--json', 'ls', '-p', folder.id]), /random.bin/) + const downloads = join(temporary, 'downloaded') + await run(['download', asset.id, '--original', '--out', downloads]) + assert.deepEqual(await readFile(join(downloads, asset.name)), bytes) + await assert.rejects( + run(['project', 'ls'], { ...cliEnv, SHUMAI_API_KEY: 'invalid-token' }), + /Unauthorized/, + ) + await writeFile( + join(temporary, '.env'), + `SHUMAI_API_SERVER=${server.url.origin}\nSHUMAI_API_KEY=fixture-token\n`, + ) + await assert.rejects( + run(['project', 'ls'], { + ...cliEnv, + SHUMAI_API_SERVER: undefined, + SHUMAI_API_KEY: undefined, + }), + /SHUMAI_API_SERVER/, + ) + await install() // Reinstall the managed binary, including a path containing spaces. + // An older compiled fixture exercises replacement and rollback on every platform. + olderDirectory = join(temporary, 'older') + const staging = join(olderDirectory, 'binary') + await mkdir(staging, { recursive: true }) + const olderSource = join(staging, 'older.ts') + await writeFile(olderSource, `console.log('${olderVersion}')\n`) + const olderBuild = await Bun.build({ + entrypoints: [olderSource], + compile: { + outfile: join(staging, windows ? 'shumai-cli.exe' : 'shumai-cli'), + autoloadDotenv: false, + autoloadBunfig: false, + }, + }) + assert.ok(olderBuild.success) + const olderArchive = packCliBinary(staging, olderDirectory, target, olderVersion) + await writeFile( + join(olderDirectory, 'checksums.txt'), + `${sha256(await readFile(olderArchive))} ${archiveName(target, olderVersion)}\n`, + ) + await install(olderVersion) + assert.equal((await run(['--version'])).trim(), olderVersion) + await install(version) + assert.equal((await run(['--version'])).trim(), version) + const installedHash = sha256(await readFile(binary)) + for (const failure of ['corrupt', 'missing', 'ambiguous'] as const) { + mode = failure + await assert.rejects(install()) + assert.equal( + sha256(await readFile(binary)), + installedHash, + `${failure} must preserve the installed executable`, + ) + } + mode = 'normal' + const marker = join(installDir, '.shumai-cli-install') + await rm(marker) + await assert.rejects(install(), /unmanaged installation/) + assert.equal(sha256(await readFile(binary)), installedHash) + console.log( + `PASS ${target}: install, API auth, mkdir, upload/download 65536 bytes, upgrade/rollback, and failure preservation`, + ) + } finally { + server.stop(true) + await rm(temporary, { recursive: true, force: true }) + } +} + +if (import.meta.main) { + const { values } = parseArgs({ options: { out: { type: 'string', default: 'out/cli-release' } } }) + await testCliRelease(resolve(values.out)) +}