release native binary: design spec and phase 1 implementation

docs(vscode-ide-companion): update vscode extension readme

package.json

@@ -23,6 +23,8 @@
     "build-and-start": "npm run build && npm run start",
     "build:vscode": "node scripts/build_vscode_companion.js",
     "build:all": "npm run build && npm run build:sandbox && npm run build:vscode",
+    "build:native": "node scripts/build_native.js",
+    "build:native:all": "node scripts/build_native.js --all",
     "build:packages": "npm run build --workspaces",
     "build:sandbox": "node scripts/build_sandbox.js",
     "bundle": "npm run generate && node esbuild.config.js && node scripts/copy_bundle_assets.js",

scripts/build_native.js

@@ -0,0 +1,323 @@
+/**
+ * @license
+ * Copyright 2025 Qwen
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const rootDir = path.resolve(__dirname, '..');
+
+const distRoot = path.join(rootDir, 'dist', 'native');
+const entryPoint = path.join(rootDir, 'packages', 'cli', 'index.ts');
+const localesDir = path.join(
+  rootDir,
+  'packages',
+  'cli',
+  'src',
+  'i18n',
+  'locales',
+);
+const vendorDir = path.join(rootDir, 'packages', 'core', 'vendor');
+
+const rootPackageJson = JSON.parse(
+  fs.readFileSync(path.join(rootDir, 'package.json'), 'utf-8'),
+);
+const cliName = Object.keys(rootPackageJson.bin || {})[0] || 'qwen';
+const version = rootPackageJson.version;
+
+const TARGETS = [
+  {
+    id: 'darwin-arm64',
+    os: 'darwin',
+    arch: 'arm64',
+    bunTarget: 'bun-darwin-arm64',
+  },
+  {
+    id: 'darwin-x64',
+    os: 'darwin',
+    arch: 'x64',
+    bunTarget: 'bun-darwin-x64',
+  },
+  {
+    id: 'linux-arm64',
+    os: 'linux',
+    arch: 'arm64',
+    bunTarget: 'bun-linux-arm64',
+  },
+  {
+    id: 'linux-x64',
+    os: 'linux',
+    arch: 'x64',
+    bunTarget: 'bun-linux-x64',
+  },
+  {
+    id: 'linux-arm64-musl',
+    os: 'linux',
+    arch: 'arm64',
+    libc: 'musl',
+    bunTarget: 'bun-linux-arm64-musl',
+  },
+  {
+    id: 'linux-x64-musl',
+    os: 'linux',
+    arch: 'x64',
+    libc: 'musl',
+    bunTarget: 'bun-linux-x64-musl',
+  },
+  {
+    id: 'windows-x64',
+    os: 'windows',
+    arch: 'x64',
+    bunTarget: 'bun-windows-x64',
+  },
+];
+
+function getHostTargetId() {
+  const platform = process.platform;
+  const arch = process.arch;
+  if (platform === 'darwin' && arch === 'arm64') return 'darwin-arm64';
+  if (platform === 'darwin' && arch === 'x64') return 'darwin-x64';
+  if (platform === 'win32' && arch === 'x64') return 'windows-x64';
+  if (platform === 'linux' && arch === 'x64') {
+    return isMusl() ? 'linux-x64-musl' : 'linux-x64';
+  }
+  if (platform === 'linux' && arch === 'arm64') {
+    return isMusl() ? 'linux-arm64-musl' : 'linux-arm64';
+  }
+  return null;
+}
+
+function isMusl() {
+  if (process.platform !== 'linux') return false;
+  const report = process.report?.getReport?.();
+  return !report?.header?.glibcVersionRuntime;
+}
+
+function parseArgs(argv) {
+  const args = {
+    all: false,
+    list: false,
+    targets: [],
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--all') {
+      args.all = true;
+    } else if (arg === '--list-targets') {
+      args.list = true;
+    } else if (arg === '--target' && argv[i + 1]) {
+      args.targets.push(argv[i + 1]);
+      i += 1;
+    } else if (arg?.startsWith('--targets=')) {
+      const raw = arg.split('=')[1] || '';
+      args.targets.push(
+        ...raw
+          .split(',')
+          .map((value) => value.trim())
+          .filter(Boolean),
+      );
+    }
+  }
+
+  return args;
+}
+
+function ensureBunAvailable() {
+  const result = spawnSync('bun', ['--version'], { stdio: 'pipe' });
+  if (result.error) {
+    console.error('Error: Bun is required to build native binaries.');
+    console.error('Install Bun from https://bun.sh and retry.');
+    process.exit(1);
+  }
+}
+
+function cleanNativeDist() {
+  fs.rmSync(distRoot, { recursive: true, force: true });
+  fs.mkdirSync(distRoot, { recursive: true });
+}
+
+function copyRecursiveSync(src, dest) {
+  if (!fs.existsSync(src)) {
+    return;
+  }
+
+  const stats = fs.statSync(src);
+  if (stats.isDirectory()) {
+    if (!fs.existsSync(dest)) {
+      fs.mkdirSync(dest, { recursive: true });
+    }
+    for (const entry of fs.readdirSync(src)) {
+      if (entry === '.DS_Store') continue;
+      copyRecursiveSync(path.join(src, entry), path.join(dest, entry));
+    }
+  } else {
+    fs.copyFileSync(src, dest);
+    if (stats.mode & 0o111) {
+      fs.chmodSync(dest, stats.mode);
+    }
+  }
+}
+
+function copyNativeAssets(targetDir, target) {
+  if (target.os === 'darwin') {
+    const sbFiles = findSandboxProfiles();
+    for (const file of sbFiles) {
+      fs.copyFileSync(file, path.join(targetDir, path.basename(file)));
+    }
+  }
+
+  copyVendorRipgrep(targetDir, target);
+  copyRecursiveSync(localesDir, path.join(targetDir, 'locales'));
+}
+
+function findSandboxProfiles() {
+  const matches = [];
+  const packagesDir = path.join(rootDir, 'packages');
+  const stack = [packagesDir];
+
+  while (stack.length) {
+    const current = stack.pop();
+    if (!current) break;
+    const entries = fs.readdirSync(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const entryPath = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(entryPath);
+      } else if (entry.isFile() && entry.name.endsWith('.sb')) {
+        matches.push(entryPath);
+      }
+    }
+  }
+
+  return matches;
+}
+
+function copyVendorRipgrep(targetDir, target) {
+  if (!fs.existsSync(vendorDir)) {
+    console.warn(`Warning: Vendor directory not found at ${vendorDir}`);
+    return;
+  }
+
+  const vendorRipgrepDir = path.join(vendorDir, 'ripgrep');
+  if (!fs.existsSync(vendorRipgrepDir)) {
+    console.warn(`Warning: ripgrep directory not found at ${vendorRipgrepDir}`);
+    return;
+  }
+
+  const platform = target.os === 'windows' ? 'win32' : target.os;
+  const ripgrepTargetDir = path.join(
+    vendorRipgrepDir,
+    `${target.arch}-${platform}`,
+  );
+  if (!fs.existsSync(ripgrepTargetDir)) {
+    console.warn(`Warning: ripgrep binaries not found at ${ripgrepTargetDir}`);
+    return;
+  }
+
+  const destVendorRoot = path.join(targetDir, 'vendor');
+  const destRipgrepDir = path.join(destVendorRoot, 'ripgrep');
+  fs.mkdirSync(destRipgrepDir, { recursive: true });
+
+  const copyingFile = path.join(vendorRipgrepDir, 'COPYING');
+  if (fs.existsSync(copyingFile)) {
+    fs.copyFileSync(copyingFile, path.join(destRipgrepDir, 'COPYING'));
+  }
+
+  copyRecursiveSync(
+    ripgrepTargetDir,
+    path.join(destRipgrepDir, path.basename(ripgrepTargetDir)),
+  );
+}
+
+function buildTarget(target) {
+  const outputName = `${cliName}-${target.id}`;
+  const targetDir = path.join(distRoot, outputName);
+  const binDir = path.join(targetDir, 'bin');
+  const binaryName = target.os === 'windows' ? `${cliName}.exe` : cliName;
+
+  fs.mkdirSync(binDir, { recursive: true });
+
+  const buildArgs = [
+    'build',
+    '--compile',
+    '--target',
+    target.bunTarget,
+    entryPoint,
+    '--outfile',
+    path.join(binDir, binaryName),
+  ];
+
+  const result = spawnSync('bun', buildArgs, { stdio: 'inherit' });
+  if (result.status !== 0) {
+    throw new Error(`Bun build failed for ${target.id}`);
+  }
+
+  const packageJson = {
+    name: outputName,
+    version,
+    os: [target.os === 'windows' ? 'win32' : target.os],
+    cpu: [target.arch],
+  };
+
+  fs.writeFileSync(
+    path.join(targetDir, 'package.json'),
+    JSON.stringify(packageJson, null, 2) + '\n',
+  );
+
+  copyNativeAssets(targetDir, target);
+}
+
+function main() {
+  if (!fs.existsSync(entryPoint)) {
+    console.error(`Entry point not found at ${entryPoint}`);
+    process.exit(1);
+  }
+
+  const args = parseArgs(process.argv.slice(2));
+  if (args.list) {
+    console.log(TARGETS.map((target) => target.id).join('\n'));
+    return;
+  }
+
+  ensureBunAvailable();
+  cleanNativeDist();
+
+  let selectedTargets = [];
+  if (args.all) {
+    selectedTargets = TARGETS;
+  } else if (args.targets.length > 0) {
+    selectedTargets = TARGETS.filter((target) =>
+      args.targets.includes(target.id),
+    );
+  } else {
+    const hostTargetId = getHostTargetId();
+    if (!hostTargetId) {
+      console.error(
+        `Unsupported host platform/arch: ${process.platform}/${process.arch}`,
+      );
+      process.exit(1);
+    }
+    selectedTargets = TARGETS.filter((target) => target.id === hostTargetId);
+  }
+
+  if (selectedTargets.length === 0) {
+    console.error('No matching targets selected.');
+    process.exit(1);
+  }
+
+  for (const target of selectedTargets) {
+    console.log(`\nBuilding native binary for ${target.id}...`);
+    buildTarget(target);
+  }
+
+  console.log('\n✅ Native build complete.');
+}
+
+main();

standalone-release.md

@@ -0,0 +1,251 @@
+# Standalone Release Spec (Bun Native + npm Fallback)
+
+This document describes the target release design for shipping Qwen Code as native
+binaries built with Bun, while retaining the existing npm JS bundle as a fallback
+distribution. It is written as a migration-ready spec that bridges the current
+release pipeline to the future dual-release system.
+
+## Goal
+
+Provide a CLI that:
+
+- Runs as a standalone binary on Linux/macOS/Windows without requiring Node or Bun.
+- Retains npm installation (global/local) as a JS-only fallback.
+- Supports a curl installer that pulls the correct binary from GitHub Releases.
+- Ships multiple variants (x64/arm64, musl/glibc where needed).
+- Uses one release flow to produce all artifacts with a single tag/version.
+
+## Non-Goals
+
+- Replacing npm as a dev-time dependency manager.
+- Shipping a single universal binary for all platforms.
+- Supporting every architecture or OS outside the defined target matrix.
+- Removing the existing Node/esbuild bundle.
+
+## Current State (Baseline)
+
+The current release pipeline:
+
+- Bundles the CLI into `dist/cli.js` via esbuild.
+- Uses `scripts/prepare-package.js` to create `dist/package.json`,
+  plus `vendor/`, `locales/`, and `*.sb` assets.
+- Publishes `dist/` to npm as the primary distribution.
+- Creates a GitHub Release and attaches only `dist/cli.js`.
+- Uses `release.yml` for nightly/preview schedules and manual stable releases.
+
+This spec extends the above pipeline; it does not replace it until the migration
+phases complete.
+
+## Target Architecture
+
+### 1) Build Outputs
+
+There are two build outputs:
+
+1. Native binaries (Bun compile) for a target matrix.
+2. Node-compatible JS bundle for npm fallback (existing `dist/` output).
+
+Native build output for each target:
+
+- dist/<name>/bin/<cli> (or .exe on Windows)
+- dist/<name>/package.json (minimal package metadata)
+
+Name encodes target:
+
+- <cli>-linux-x64
+- <cli>-linux-x64-musl
+- <cli>-linux-arm64
+- <cli>-linux-arm64-musl
+- <cli>-darwin-arm64
+- <cli>-darwin-x64
+- <cli>-windows-x64
+
+### 2) npm Distribution (JS Fallback)
+
+Keep npm as a pure JS/TS CLI package that runs under Node/Bun. Do not ship or
+auto-install native binaries through npm.
+
+Implications:
+
+- npm install always uses the JS implementation.
+- No optionalDependencies for platform binaries.
+- No postinstall symlink logic.
+- No node shim that searches for a native binary.
+
+### 3) GitHub Release Distribution (Primary)
+
+Native binaries are distributed only via GitHub Releases and the curl installer:
+
+- Archive each platform binary into a tar.gz (Linux) or zip (macOS/Windows).
+- Attach archives to the GitHub Release.
+- Provide a shell installer that detects target and downloads the correct archive.
+
+## Detailed Implementation
+
+### A) Target Matrix
+
+Define a target matrix that includes OS, arch, and libc variants.
+
+Target list (fixed set):
+
+- darwin arm64
+- darwin x64
+- linux arm64 (glibc)
+- linux x64 (glibc)
+- linux arm64 musl
+- linux x64 musl
+- win32 x64
+
+### B) Build Scripts
+
+1. Native build script (new, e.g. `scripts/build-native.ts`)
+   Responsibilities:
+
+- Remove native build output directory (keep npm `dist/` intact).
+- For each target:
+  - Compute a target name.
+  - Compile using `Bun.build({ compile: { target: ... } })`.
+  - Write the binary to `dist/<name>/bin/<cli>`.
+  - Write a minimal `package.json` into `dist/<name>/`.
+
+2. npm fallback build (existing)
+   Responsibilities:
+
+- `npm run bundle` produces `dist/cli.js`.
+- `npm run prepare:package` creates `dist/package.json` and copies assets.
+
+Key details:
+
+- Use Bun.build with compile.target = <bun-target> (e.g. bun-linux-x64).
+- Include any extra worker/runtime files in entrypoints.
+- Use define or execArgv to inject version/channel metadata.
+- Use "windows" in archive naming even though the OS is "win32" internally.
+
+Build-time considerations:
+
+- Preinstall platform-specific native deps for bundling (example: bun install --os="_" --cpu="_" for dependencies with native bindings).
+- Include worker assets in the compile entrypoints and embed their paths via define constants.
+- Use platform-specific bunfs root paths when resolving embedded worker files.
+- Set runtime execArgv flags for user-agent/version and system CA usage.
+
+Target name example:
+<cli>-<os>-<arch>[-musl]
+
+Minimal package.json example:
+{
+"name": "<cli>-linux-x64",
+"version": "<version>",
+"os": ["linux"],
+"cpu": ["x64"]
+}
+
+### C) Publish Script (new, optional)
+
+Responsibilities:
+
+1. Run the native build script.
+2. Smoke test a local binary (`dist/<host>/bin/<cli> --version`).
+3. Create GitHub Release archives.
+4. Optionally build and push Docker image.
+5. Publish npm package (JS-only fallback) as a separate step or pipeline.
+
+Note: npm publishing is now independent of native binary publishing. It should not reference platform binaries.
+
+### D) GitHub Release Installer (install)
+
+A bash installer that:
+
+1. Detects OS and arch.
+2. Handles Rosetta (macOS) and musl detection (Alpine, ldd).
+3. Builds target name and downloads from GitHub Releases.
+4. Extracts to ~/.<cli>/bin.
+5. Adds PATH unless --no-modify-path.
+
+Supports:
+
+- --version <version>
+- --binary <path>
+- --no-modify-path
+
+Installer details to include:
+
+- Require tar for Linux and unzip for macOS/Windows archives.
+- Use "windows" in asset naming, not "win32".
+- Prefer arm64 when macOS is running under Rosetta.
+
+## CI/CD Flow (Dual Pipeline)
+
+Release pipeline (native binaries):
+
+1. Bump version.
+2. Build binaries for the full target matrix.
+3. Smoke test the host binary.
+4. Create GitHub release assets.
+5. Mark release as final (if draft).
+
+Release pipeline (npm fallback):
+
+1. Bump version (same tag).
+2. Publish the JS-only npm package.
+
+Release orchestration details to consider:
+
+- Update all package.json version fields in the repo.
+- Update any extension metadata or download URLs that embed version strings.
+- Tag the release and create a GitHub Release draft that includes the binary assets.
+
+### Workflow Mapping to Current Code
+
+The existing `release.yml` workflow remains the orchestrator:
+
+- Use `scripts/get-release-version.js` for version/tag selection.
+- Keep tests and integration checks as-is.
+- Add a native build matrix job that produces archives and uploads them to
+  the GitHub Release.
+- Keep the npm publish step from `dist/` as the fallback.
+- Ensure the same `RELEASE_TAG` is used for both native and npm outputs.
+
+## Edge Cases and Pitfalls
+
+- musl: Alpine requires musl binaries.
+- Rosetta: macOS under Rosetta should prefer arm64 when available.
+- npm fallback: ensure JS implementation is functional without native helpers.
+- Path precedence: binary install should appear earlier in PATH than npm global bin if you want native to win by default.
+- Archive prerequisites: users need tar/unzip depending on OS.
+
+## Testing Plan
+
+- Build all targets in CI.
+- Run dist/<host>/bin/<cli> --version.
+- npm install locally and verify CLI invocation.
+- Run installer script on each OS or VM.
+- Validate musl builds on Alpine.
+
+## Migration Plan
+
+Phase 1: Add native builds without changing npm
+
+- [ ] Define target matrix with musl variants.
+- [ ] Add native build script for Bun compile per target.
+- [ ] Generate per-target package.json.
+- [ ] Produce per-target archives and upload to GitHub Releases.
+- [ ] Keep existing npm bundle publish unchanged.
+
+Phase 2: Installer and docs
+
+- [ ] Add curl installer for GitHub Releases.
+- [ ] Document recommended install paths (native first).
+- [ ] Add smoke tests for installer output.
+
+Phase 3: Default install guidance and cleanup
+
+- [ ] Update docs to recommend native install where possible.
+- [ ] Decide whether npm stays equal or fallback-only in user docs.
+
+## Implementation Checklist
+
+- [ ] Keep `npm run bundle` + `npm run prepare:package` for JS fallback.
+- [ ] Add `scripts/build-native.ts` for Bun compile targets.
+- [ ] Add archive creation and asset upload in `release.yml`.
+- [ ] Add an installer script with OS/arch/musl detection.
+- [ ] Ensure tag/version parity across native and npm releases.