From 80f3128282da34162fc9bbe6862feb50743cc678 Mon Sep 17 00:00:00 2001 From: Fafo Date: Wed, 2 Sep 2026 10:17:56 +0200 Subject: [PATCH 01/23] feat: carry MCP servers, linked home scripts, memories and external tools (v0.3) `arbella pull` restored only the obvious files. This makes it plug and play: - Lift `~/.claude.json#mcpServers` (+ per-project) into the manifest, sanitized and templated; merge back key-wise on pull (atomic write, safety copy, other keys untouched). Dry run prints each launch command. - New `shared/home/` root for files under $HOME outside every tool home: scripts referenced by hook/statusLine/MCP commands (single hop, files only, 1 MiB cap, realpath under $HOME), user `extraPaths` (`arbella init --extra-paths`), and the claude-mem companion config. `shared/home-index.json` records provenance so a push from a machine that lacks a tool never deletes that tool's files. Restore refuses symlinked path components and never writes outside $HOME. - Claude per-project memories under `claude/memories/` (HOME slug re-derived per machine), gated by `includeMemories`. - `manifest.externalTools`: brew / uv tool / pipx binaries behind MCP and hook commands, reinstalled best-effort; names validated at parse, collect and install time. - Claude FROZEN_PATHS += rules, scripts, output-styles, keybindings.json, mcp-configs, AGENTS.md; root-anchored denylist entries, *.bak*, .venv/; HOME_DENY for carried home files; credential-bearing MCP URLs redacted. - Codex local `[marketplaces.*]` sources templated (leaked absolute paths) and skipped on restore when missing. - status: MCP / external-tool drift, shared/home diff, unmanaged hint. Manifest schema version stays 1 (all new fields defaulted). Tests: 230 -> 490. Verified with a sandboxed push/status/pull --dry-run against a real machine; three adversarial review passes applied. Claude-Session: https://claude.ai/code/session_01HN68sBfxL6MeEVVQ3T8Xq1 --- .gitignore | 4 + README.md | 43 +- package.json | 4 +- src/adapters/antigravity/index.ts | 17 +- src/adapters/claude/capture.ts | 187 ++++- src/adapters/claude/mcp.ts | 639 +++++++++++++++ src/adapters/claude/memories.ts | 210 +++++ src/adapters/claude/paths.ts | 81 +- src/adapters/claude/plugins.ts | 6 +- src/adapters/claude/restore.ts | 110 ++- src/adapters/codex/capture.ts | 98 ++- src/adapters/codex/configToml.ts | 18 +- src/adapters/codex/restore.ts | 24 +- src/adapters/cursor/index.ts | 17 +- src/adapters/shared/configDir.ts | 18 +- src/commands/backup.ts | 303 ++++++- src/commands/init.ts | 99 ++- src/commands/restore.ts | 337 +++++++- src/commands/status.ts | 213 ++++- src/core/config/schema.ts | 8 + src/core/externaltools/classify.ts | 268 +++++++ src/core/externaltools/collect.ts | 267 +++++++ src/core/homefiles/capture.ts | 668 ++++++++++++++++ src/core/homefiles/home-index.ts | 157 ++++ src/core/homefiles/restore.ts | 210 +++++ src/core/homefiles/scan.ts | 327 ++++++++ src/core/homefiles/suggest.ts | 163 ++++ src/core/manifest/index.ts | 63 +- src/core/manifest/schema.ts | 84 ++ src/core/sanitizer/denylist.ts | 82 +- src/core/sanitizer/index.ts | 30 +- src/core/sanitizer/patterns.ts | 92 +++ src/platform/install.ts | 138 ++++ src/types.ts | 21 + src/utils/fs.ts | 56 ++ src/utils/object.ts | 44 ++ src/utils/safe-path.ts | 58 ++ test/integration/capture-restore.test.ts | 569 ++++++++++++- test/unit/claude-mcp.test.ts | 601 ++++++++++++++ test/unit/claude-memories.test.ts | 425 ++++++++++ .../cursor-cli-and-antigravity-wiring.test.ts | 10 +- test/unit/denylist.test.ts | 171 ++++ test/unit/externaltools-classify.test.ts | 403 ++++++++++ test/unit/externaltools-collect.test.ts | 482 +++++++++++ test/unit/homefiles-capture.test.ts | 747 ++++++++++++++++++ test/unit/homefiles-index.test.ts | 162 ++++ test/unit/homefiles-restore.test.ts | 138 ++++ test/unit/homefiles-scan.test.ts | 278 +++++++ test/unit/homefiles-suggest.test.ts | 108 +++ test/unit/manifest-templating.test.ts | 112 ++- test/unit/manifest.test.ts | 47 ++ test/unit/sanitizer.test.ts | 117 +++ test/unit/shared-home-mirror.test.ts | 202 +++++ 53 files changed, 9553 insertions(+), 183 deletions(-) create mode 100644 src/adapters/claude/mcp.ts create mode 100644 src/adapters/claude/memories.ts create mode 100644 src/core/externaltools/classify.ts create mode 100644 src/core/externaltools/collect.ts create mode 100644 src/core/homefiles/capture.ts create mode 100644 src/core/homefiles/home-index.ts create mode 100644 src/core/homefiles/restore.ts create mode 100644 src/core/homefiles/scan.ts create mode 100644 src/core/homefiles/suggest.ts create mode 100644 src/utils/object.ts create mode 100644 src/utils/safe-path.ts create mode 100644 test/unit/claude-mcp.test.ts create mode 100644 test/unit/claude-memories.test.ts create mode 100644 test/unit/denylist.test.ts create mode 100644 test/unit/externaltools-classify.test.ts create mode 100644 test/unit/externaltools-collect.test.ts create mode 100644 test/unit/homefiles-capture.test.ts create mode 100644 test/unit/homefiles-index.test.ts create mode 100644 test/unit/homefiles-restore.test.ts create mode 100644 test/unit/homefiles-scan.test.ts create mode 100644 test/unit/homefiles-suggest.test.ts create mode 100644 test/unit/shared-home-mirror.test.ts diff --git a/.gitignore b/.gitignore index 39bd9d6..9e612d9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,10 @@ Thumbs.db .idea/ .vscode/ +# Agent tooling +.serena/ +.claude/ + # Env / local .env .env.local diff --git a/README.md b/README.md index 09d13e2..0c1dfba 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ memories — into a private Git repo you own. [![License](https://img.shields.io/badge/license-AGPL--3.0--only-blue.svg)](./LICENSE)  ![Node](https://img.shields.io/badge/node-%E2%89%A518-43853d?logo=node.js&logoColor=white)  ![Platforms](https://img.shields.io/badge/platform-Linux%20%C2%B7%20macOS%20%C2%B7%20Windows-555) - ![Status](https://img.shields.io/badge/status-v0.1-orange) + ![Status](https://img.shields.io/badge/status-v0.3-orange) -[Quick start](#quick-start) · [Commands](#commands) · [How sign-in works](#how-sign-in-works) · [Secrets](#secrets) +[Quick start](#quick-start) · [Commands](#commands) · [How sign-in works](#how-sign-in-works) · [What still needs you after a pull](#what-still-needs-you-after-a-pull) · [Secrets](#secrets) @@ -51,18 +51,28 @@ arbella update # later: update arbella itself ## What it does ```text - push ~/.claude · ~/.codex · Cursor ──▶ strip · template ──▶ private repo - pull private repo ──▶ install · place ──▶ tools + Cursor User data + push ~/.claude · ~/.codex · Cursor · $HOME ──▶ strip · template ──▶ private repo + pull private repo ──▶ install · place ──▶ tools + Cursor User data + $HOME ``` -`push` reads the parts of your setup that matter, strips anything secret, swaps machine-specific paths for placeholders, and pushes the result to your repo. `pull` does the reverse: installs the CLIs you don't have, drops files back with this machine's paths, reinstalls plugins and skills, and wires shared Claude/Codex instructions back into `CLAUDE.md` and `AGENTS.md`. +`push` reads the parts of your setup that matter, strips anything secret, swaps machine-specific paths for placeholders, and pushes the result to your repo. `pull` does the reverse: installs the CLIs you don't have, drops files back with this machine's paths, merges MCP servers into `~/.claude.json`, writes shared home files back under `$HOME`, reinstalls plugins, skills, and external tools, and wires shared Claude/Codex instructions back into `CLAUDE.md` and `AGENTS.md`. + +Beyond the obvious config files, Arbella also carries: + +| It also carries | Where it comes from | Notes | +| --- | --- | --- | +| MCP servers | `~/.claude.json` → `mcpServers` | Never the whole file — only the server definitions, sanitized and templated; env values are redacted unless `includeSecrets` is on | +| The rest of `~/.claude` | `rules/`, `scripts/`, `output-styles/`, `keybindings.json`, `mcp-configs/`, `AGENTS.md` | The pieces your hooks and `CLAUDE.md` actually reference | +| Shared home files | `shared/home/` | Scripts your hooks, statusLine, or MCP commands point at outside any tool's home — picked up automatically (one hop, files only, 1 MiB cap) — plus anything you list in `extraPaths`, and the claude-mem companion `~/.claude-mem/settings.json` | +| Claude memories | `claude/memories/` | Per-project memory, only when `includeMemories` is on | +| External tools | manifest `externalTools` | Binaries behind an MCP/hook command that brew, `uv tool`, or pipx installed — reinstalled best-effort on pull | Two things it deliberately won't do: - **Never commits secrets.** API keys, OAuth tokens, `auth.json`, `.credentials.json` — all excluded, no exceptions. You sign back in after a pull, or carry them yourself with [`arbella secrets`](#arbella-secrets). - **Doesn't copy what it can reinstall.** Plugins and registry skills are saved as a list and pulled fresh, so the repo stays small and never goes stale. -Supported today: **Claude Code**, **Codex**, **Cursor** (IDE + the `cursor-agent` CLI), **opencode**, **GitHub Copilot CLI**, **Kilo Code**, and **Google Antigravity**. Cursor support covers global MCP config, user settings, keybindings, snippets, local skills, skills.sh symlinks, the `cursor-agent` CLI config + global commands, and extension IDs; runtime state and credentials stay out. opencode, Copilot CLI, and Kilo are config-dir CLIs — Arbella captures each one's config dir (the JSON/JSONC config, custom agents/commands, MCP config), strips token-shaped values, and skips machine-local state, plugin install artifacts, and reinstallable skills (regenerated from the config on first run). Antigravity spans three roots — its VS Code-style User settings, the `~/.antigravity` extension list, and the shared `~/.gemini` agent config (global rules, MCP servers, skills) — while its Google OAuth tokens never leave your machine. +Supported today: **Claude Code**, **Codex**, **Cursor** (IDE + the `cursor-agent` CLI), **opencode**, **GitHub Copilot CLI**, **Kilo Code**, and **Google Antigravity**. Claude Code coverage also reaches into `~/.claude.json` for MCP servers, per-project memories, and any linked scripts living under `$HOME`. Cursor support covers global MCP config, user settings, keybindings, snippets, local skills, skills.sh symlinks, the `cursor-agent` CLI config + global commands, and extension IDs; runtime state and credentials stay out. opencode, Copilot CLI, and Kilo are config-dir CLIs — Arbella captures each one's config dir (the JSON/JSONC config, custom agents/commands, MCP config), strips token-shaped values, and skips machine-local state, plugin install artifacts, and reinstallable skills (regenerated from the config on first run). Antigravity spans three roots — its VS Code-style User settings, the `~/.antigravity` extension list, and the shared `~/.gemini` agent config (global rules, MCP servers, skills) — while its Google OAuth tokens never leave your machine. ## Commands @@ -100,6 +110,7 @@ The one-time wiring. Pick a host (GitHub, GitLab, or a plain Git URL); if the re ```sh arbella init arbella init --provider github --repo you/your-setup --auto-push daily -y +arbella init --extra-paths ~/.agents/hooks,~/.agents/memory -y ``` Saves your preferences: @@ -108,6 +119,7 @@ Saves your preferences: - who wins a conflict — your machine or the repo - auto-push cadence — off, once per session, or daily - whether secrets are allowed into the repo (off by default), and whether to include memories (also off) +- extra `$HOME` paths to carry along (`--extra-paths`, comma-separated, or the interactive prompt) — Arbella pre-fills the prompt with suggestions from the scripts your hooks and MCP servers already point at. Suggestions are never accepted on your behalf: `-y` keeps whatever is already configured, so pass `--extra-paths` to carry them non-interactively ### `arbella push` @@ -132,6 +144,9 @@ arbella pull --dry-run - First copies your current `~/.claude` and `~/.codex` to a timestamped safety folder, so a pull can't quietly wreck what's already there. - Installs any missing CLIs — reaching for `sudo` only when the global npm folder actually needs root. - Writes files with this machine's paths, then reinstalls your plugins and skills from the manifest. +- Merges MCP servers into `~/.claude.json` — a safety copy is taken first, and only the `mcpServers` keys are touched; your other keys (auth, telemetry) are never read or written. +- Writes shared home files (linked scripts, `extraPaths`) back under `$HOME` — an existing file is snapshotted first and never overwritten while your machine is the source of truth. +- Reinstalls external tools behind MCP/hook commands via brew, `uv tool`, or pipx, best-effort. - Deploys your shared instructions to `CLAUDE.md` and `AGENTS.md`. - Reminds you to sign back in at the end, since no credentials came along for the ride. @@ -144,7 +159,7 @@ arbella status arbella status --json # pipe it somewhere ``` -New and modified files, plugin drift, the secrets it would skip. Writes nothing, installs nothing. +New and modified files, plugin drift, MCP server and external-tool drift, and the secrets it would skip. Files it doesn't recognize print as `not backed up (unknown to arbella)` — add them via `extraPaths` or open an issue. Writes nothing, installs nothing. ### `arbella update` @@ -188,12 +203,24 @@ Short version: **gh and glab are preferred, not required.** They're the smoothes For the device flow, point Arbella at your own registered OAuth app with `ARBELLA_GITHUB_CLIENT_ID` (or `ARBELLA_GITLAB_CLIENT_ID`). There's no sensible default to ship — the client ID is public but tied to your account — so with none set, Arbella skips the device flow and asks for a token instead. SSH and `file://` remotes are left alone, so those keep working however your Git already has them. +## What still needs you after a pull + +Arbella restores everything it safely can. A few things are yours to finish: + +- **Credentials.** No API keys or OAuth tokens travel with a pull — sign back in when Arbella tells you to, or carry them yourself with [`arbella secrets`](#arbella-secrets). +- **Redacted MCP env values.** If an MCP server's config held a secret-shaped value, push replaced it with `{{REDACTED}}`. Pull prints one line per key it couldn't restore, e.g. `claude: MCP server serena needs env SERENA_TOKEN re-supplied (redacted on backup)`. +- **External tools Arbella couldn't classify or install.** brew/`uv tool`/pipx tools are reinstalled best-effort; anything unknown, or that failed, is listed by name and by what uses it, so you can install it yourself. +- **`~/.claude.json` is only merged, key by key.** Pull touches `mcpServers` (and per-project `mcpServers`) and nothing else in that file — your `oauthAccount`, telemetry, and everything else stay exactly as they were. +- **The repo is trusted.** A pull writes the hooks, MCP commands, and install lists your repo describes, and runs `brew` / `uv tool` / `pipx` / `npm` for the tools it lists — exactly like the machine you pushed from. Keep the repo private, and run `arbella pull --dry-run` first on anything you did not push yourself: the plan prints every command and target path. + ## Secrets This is the part I most wanted to get right. - Whole credential files (`auth.json`, `.credentials.json`, and the like) sit on a hard denylist and are never read into the repo. - Everything that *does* get committed runs through a sanitizer that redacts token-shaped values. +- `~/.claude.json` itself is still never stored — only its `mcpServers` sub-objects are lifted out, sanitized, and templated; the rest of that file (OAuth account, telemetry) never leaves your machine. +- Shared home files go through the same sanitizer, plus a home-specific denylist — `.env`, `*.pem`, `*.key`, `id_rsa*`, `.netrc`, `.npmrc`, `.git-credentials`, and friends are never captured even when something points at them. - Any token Arbella holds for itself lives in one local file with `0600` permissions — not in the repo, not in a logged command, not baked into a Git remote. - A test fails the build if a credential can reach the repo. This isn't on the honor system. @@ -209,7 +236,7 @@ The Arbella was the flagship of the 1630 Winthrop Fleet — a few hundred settle ## Status -v0.1 — early, but it builds clean, the full test suite passes, and I use it daily. Issues and pull requests welcome. +v0.3 — early, but it builds clean, the full test suite passes, and I use it daily. Issues and pull requests welcome. ## License diff --git a/package.json b/package.json index b53cc1d..fff885b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "arbella", - "version": "0.2.0", - "description": "Set up your AI coding tools once, then rebuild that setup on any machine. Claude Code, Codex, Cursor, opencode, GitHub Copilot CLI, Kilo Code & Antigravity — Linux, macOS, Windows.", + "version": "0.3.0", + "description": "Set up your AI coding tools once, then rebuild that setup on any machine — MCP servers included. Claude Code, Codex, Cursor, opencode, GitHub Copilot CLI, Kilo Code & Antigravity — Linux, macOS, Windows.", "type": "module", "license": "AGPL-3.0-only", "author": "Fafoooo", diff --git a/src/adapters/antigravity/index.ts b/src/adapters/antigravity/index.ts index 1193c53..664df29 100644 --- a/src/adapters/antigravity/index.ts +++ b/src/adapters/antigravity/index.ts @@ -35,10 +35,10 @@ import type { OS, RestoreAction, SecretRef, - ToolManifest, } from "../../types.js"; import { denylistFor, matchesDeny } from "../../core/sanitizer/denylist.js"; +import { emptyManifest } from "../../core/manifest/index.js"; import { cliBinaryName, detectOS, installCommandFor } from "../../platform/os.js"; import { runInstall, which } from "../../platform/install.js"; import { normalizeCapturedSymlinkTarget } from "../../utils/symlink.js"; @@ -139,19 +139,6 @@ function targetAbsFor(dirs: RootDirs, target: AntigravityTarget): string { /* Small helpers */ /* -------------------------------------------------------------------------- */ -/** An empty manifest for antigravity (it tracks no reinstallable plugins here — - * the extension list is frozen as a file, not reinstalled from a manifest). */ -function emptyManifest(): ToolManifest { - return { - tool: "antigravity", - plugins: [], - marketplaces: [], - skills: [], - npmGlobals: [], - enabledPlugins: {}, - }; -} - /** Read mode bits best-effort so executable helper files keep their mode. */ async function readMode(abs: string): Promise { try { @@ -291,7 +278,7 @@ export async function capture(ctx: CaptureContext): Promise { const symlinks: CapturedSymlink[] = []; const secrets: SecretRef[] = []; const warnings: string[] = []; - const manifest = emptyManifest(); + const manifest = emptyManifest("antigravity"); const deny = denylistFor("antigravity"); const roots = await captureRoots(ctx); diff --git a/src/adapters/claude/capture.ts b/src/adapters/claude/capture.ts index 4003b94..338a20d 100644 --- a/src/adapters/claude/capture.ts +++ b/src/adapters/claude/capture.ts @@ -19,7 +19,18 @@ * - Collect npm globals via the shared platform helper. * - Any denylisted-but-present secret file (.credentials.json, .claude.json) * is recorded as a SecretRef(kind:"file") so the user is reminded to - * re-supply it; its CONTENTS are never read or stored. + * re-supply it; its CONTENTS are never read or stored. The ONE exception is + * ~/.claude.json's `mcpServers` / `projects.*.mcpServers` sub-objects, which + * mcp.ts lifts into the manifest (sanitized + templated) — no other key of + * that file is ever read past the parse. + * - projects//memory/** is captured under its own `claude/memories/` + * root when ctx.includeMemories is on (see memories.ts). + * - Scripts the configs POINT AT but that live outside ~/.claude (hook + * dispatchers, statusline scripts, MCP launchers under ~/.agents, ~/.local/bin) + * are followed with core/homefiles and emitted as `shared/home/` files + * inside this same CaptureResult, together with the well-known plugin + * companion configs (~/.claude-mem/settings.json). The backup command merges + * every tool's shared/home files into one mirrored root. * * All fs/sanitizer/templater work goes through the injected CaptureContext, so * this module is unit-testable against a fixture dir. No clock, no direct @@ -42,30 +53,58 @@ import type { } from "../../types.js"; import { + COMMON_DENY, denylistFor, matchesDeny, } from "../../core/sanitizer/denylist.js"; +import type { CommandRef } from "../../core/homefiles/scan.js"; +import { collectCommandRefs } from "../../core/homefiles/scan.js"; +import { + captureHomeFile, + captureLinkedHomeFiles, + homeExcludeRoots, +} from "../../core/homefiles/capture.js"; +import { + capturedAbsolutePaths, + collectExternalTools, +} from "../../core/externaltools/collect.js"; import { normalizeCapturedSymlinkTarget } from "../../utils/symlink.js"; import { listNpmGlobals } from "../../platform/install.js"; +import type { ClaudePaths } from "./paths.js"; import { REPO_PREFIX, FROZEN_PATHS, INSTRUCTIONS_FILE, paths } from "./paths.js"; import { parseInstalledPlugins, parseKnownMarketplaces, extractEnabledPlugins, } from "./plugins.js"; +import { captureMcpServers } from "./mcp.js"; +import { captureMemories, slugifyPath } from "./memories.js"; -/** Files known to hold whole-file secrets; recorded (never read) if present. */ -const SECRET_FILES: ReadonlyArray<{ rel: string; key: string; description: string }> = [ +/** + * Files known to hold whole-file secrets; recorded (never read) if present. + * `abs` resolves each one from the path set because they do not all live INSIDE + * the tool home — ~/.claude.json is a sibling of ~/.claude. + */ +const SECRET_FILES: ReadonlyArray<{ + rel: string; + key: string; + description: string; + abs: (p: ClaudePaths) => string; +}> = [ { rel: ".credentials.json", key: ".credentials.json", description: "Claude OAuth / API credentials — re-login after restore.", + abs: (p) => path.join(p.home, ".credentials.json"), }, { rel: ".claude.json", key: ".claude.json", - description: "Claude global state (auth tokens + project history) — excluded.", + description: + "Claude global state (auth + telemetry) — excluded; MCP server definitions " + + "are carried in manifest.mcpServers.", + abs: (p) => p.globalState, }, ]; @@ -121,15 +160,22 @@ async function walk( } // kind === "file" - await captureFile(ctx, home, abs, rel, files, warnings); + await captureFile(ctx, abs, repoPathFor(rel), rel, files, warnings); } -/** Read + sanitize + template a single file and push a CapturedFile. */ +/** + * Read + sanitize + template a single file and push a CapturedFile. + * + * `repoPath` is the destination inside the backup repo and `source` the label + * used for sanitizer SecretRefs and warnings. They are passed explicitly (rather + * than derived from a tool-home-relative path) because memories land under a + * different repo root than the frozen tree — see memories.ts. + */ async function captureFile( ctx: CaptureContext, - home: string, abs: string, - rel: string, + repoPath: string, + source: string, files: CapturedFile[], warnings: string[], ): Promise { @@ -147,7 +193,7 @@ async function captureFile( try { bytes = await ctx.fs.readBytes(abs); } catch (err) { - warnings.push(`claude: could not read ${rel}: ${(err as Error).message}`); + warnings.push(`claude: could not read ${source}: ${(err as Error).message}`); return; } @@ -161,13 +207,13 @@ async function captureFile( // a warning rather than base64'ing it in raw. if ( !ctx.includeSecrets && - binaryScanViews(bytes).some((view) => ctx.sanitizer.sanitizeText(view, "claude", rel).changed) + binaryScanViews(bytes).some((view) => ctx.sanitizer.sanitizeText(view, "claude", source).changed) ) { - warnings.push(`claude: skipped ${rel} — binary content with secret-shaped bytes`); + warnings.push(`claude: skipped ${source} — binary content with secret-shaped bytes`); return; } const file: CapturedFile = { - repoPath: repoPathFor(rel), + repoPath, content: bytes.toString("base64"), binary: true, }; @@ -194,11 +240,13 @@ async function captureFile( // UTF-16 files are decoded to text above, so a lone NUL byte never routes a // secret-bearing config around sanitizeFile. const raw = decoded.text; - const content = ctx.includeSecrets ? raw : ctx.sanitizer.sanitizeFile(raw, "claude", rel).content; + const content = ctx.includeSecrets + ? raw + : ctx.sanitizer.sanitizeFile(raw, "claude", source).content; const templated = ctx.templater.toTemplate(content, ctx.vars); const file: CapturedFile = { - repoPath: repoPathFor(rel), + repoPath, content: templated, }; if (mode !== undefined) file.mode = mode; @@ -251,11 +299,56 @@ async function captureSkills( ctx.log.debug(`claude: skill (frozen) ${name}`); } else if (kind === "file") { // A loose file directly under skills/ (rare) — freeze it. - await captureFile(ctx, home, abs, rel, files, warnings); + await captureFile(ctx, abs, repoPathFor(rel), rel, files, warnings); } } } +/** + * Companion config dirs of Claude PLUGINS: reinstalled from the manifest, but + * their own settings live outside ~/.claude and would otherwise be lost. Paths + * are $HOME-relative POSIX and land under `shared/home/` like any linked script. + */ +const COMPANION_FILES: ReadonlyArray<{ rel: string; reason: string }> = [ + { rel: ".claude-mem/settings.json", reason: "companion:claude-mem" }, +]; + +/** + * Capture everything Claude needs that lives in $HOME but OUTSIDE ~/.claude: + * the hook/statusline/MCP scripts its configs point at, and the well-known + * plugin companion configs. + * + * The command refs come from the already-parsed settings files and from + * mcp.ts's raw pass over ~/.claude.json — no config file is read twice, and no + * key of ~/.claude.json beyond its MCP sub-objects is ever looked at. + * + * Returns the full ref list so the external-tool collector (WP-C) can classify + * the same commands without parsing anything again. + */ +async function captureHomeExtras( + ctx: CaptureContext, + settingsJson: unknown, + settingsLocalJson: unknown, + mcpRefs: readonly CommandRef[], + out: { files: CapturedFile[]; secrets: SecretRef[]; warnings: string[] }, +): Promise { + const opts = { excludeRoots: homeExcludeRoots(ctx.toolHome) }; + + const refs: CommandRef[] = [ + ...collectCommandRefs(settingsJson, "claude:settings.json"), + ...collectCommandRefs(settingsLocalJson, "claude:settings.local.json"), + ...mcpRefs, + ]; + await captureLinkedHomeFiles(ctx, refs, "claude", out, opts); + + for (const companion of COMPANION_FILES) { + const abs = path.join(ctx.vars.HOME, ...companion.rel.split("/")); + await captureHomeFile(ctx, abs, "claude", companion.reason, out, opts); + } + + return refs; +} + /** Read + parse a JSON file via the fs service; returns undefined if absent/bad. */ async function readJson( ctx: CaptureContext, @@ -308,10 +401,36 @@ export async function capture( await walk(ctx, home, path.join(home, top), deny, files, symlinks, warnings); } + // ----- Per-project memories (R13, opt-in) ----- + // Stored under their OWN repo root (claude/memories/…), not claude/files/, + // because ~/.claude/projects is denylisted wholesale and the layout has to be + // portable across machines with different $HOME values. + if (ctx.includeMemories) { + await captureMemories( + ctx, + p.projectsDir, + slugifyPath(ctx.vars.HOME), + // COMMON_DENY, not the Claude list: memory paths are relative to a + // project's memory/ dir, so the root-anchored Claude patterns ("/feedback/", + // "/plugins/") would match a memory SUBDIRECTORY of the same name and drop + // it. Only the universal cruft rules make sense at this root. + [...COMMON_DENY], + (abs, repoPath, source) => captureFile(ctx, abs, repoPath, source, files, warnings), + ); + } else { + ctx.log.debug("claude: skipping projects/*/memory (includeMemories is off)"); + } + // ----- Manifest: plugins, marketplaces, enabledPlugins ----- const installedJson = await readJson(ctx, p.installedPlugins, warnings, "installed_plugins.json"); const marketplacesJson = await readJson(ctx, p.knownMarketplaces, warnings, "known_marketplaces.json"); const settingsJson = await readJson(ctx, p.settings, warnings, "settings.json"); + const settingsLocalJson = await readJson( + ctx, + p.settingsLocal, + warnings, + "settings.local.json", + ); // Fold project-scope plugin paths (projectPath) to {{HOME}}-style placeholders, // exactly like file contents — otherwise the raw machine path leaks into the repo. @@ -338,8 +457,7 @@ export async function capture( // ----- Secret files: record presence, never read contents ----- for (const sf of SECRET_FILES) { - const abs = path.join(home, sf.rel); - if ((await ctx.fs.exists(abs))) { + if (await ctx.fs.exists(sf.abs(p))) { secrets.push({ tool: "claude", source: sf.rel, @@ -350,6 +468,34 @@ export async function capture( } } + // ----- MCP servers lifted out of ~/.claude.json (see mcp.ts) ----- + const mcp = await captureMcpServers(ctx); + secrets.push(...mcp.secrets); + warnings.push(...mcp.warnings); + + // ----- Linked scripts + companion configs in $HOME (WP-B) ----- + const commandRefs = await captureHomeExtras( + ctx, + settingsJson, + settingsLocalJson, + mcp.commandRefs, + { files, secrets, warnings }, + ); + + // ----- External tools behind those same commands (WP-C) ----- + // Runs on the FULL file list so a command pointing at something this capture + // already carries (a statusline script under ~/.claude, a hook dispatcher in + // shared/home) is restored as a file rather than reported as a missing package. + const externalTools = await collectExternalTools(commandRefs, { + home: ctx.vars.HOME, + capturedPaths: capturedAbsolutePaths(files, { + home: ctx.vars.HOME, + toolHome: home, + filesPrefix: REPO_PREFIX, + }), + toTemplate: (value) => ctx.templater.toTemplate(value, ctx.vars), + }); + const manifest: ToolManifest = { tool: "claude", plugins, @@ -357,12 +503,17 @@ export async function capture( skills, npmGlobals, enabledPlugins, + mcpServers: mcp.mcpServers, + projectMcpServers: mcp.projectMcpServers, + externalTools, }; ctx.log.debug( `claude: captured ${files.length} files, ${symlinks.length} symlinks, ` + `${plugins.length} plugins, ${marketplaces.length} marketplaces, ` + - `${skills.length} skills, ${npmGlobals.length} npm globals`, + `${skills.length} skills, ${npmGlobals.length} npm globals, ` + + `${Object.keys(mcp.mcpServers).length} MCP servers, ` + + `${externalTools.length} external tools`, ); return { tool: "claude", files, symlinks, manifest, secrets, warnings }; diff --git a/src/adapters/claude/mcp.ts b/src/adapters/claude/mcp.ts new file mode 100644 index 0000000..7060d96 --- /dev/null +++ b/src/adapters/claude/mcp.ts @@ -0,0 +1,639 @@ +/** + * MCP server definitions from ~/.claude.json — the one thing arbella lifts OUT of + * Claude's global-state file. + * + * ~/.claude.json is denylisted wholesale and always will be: it holds the OAuth + * account, telemetry counters and the full project history. But it is ALSO the + * only place user-scope MCP servers live, and losing them is losing the setup. + * So this module reads exactly two sub-objects out of it — + * + * mcpServers -> manifest.mcpServers + * projects..mcpServers -> manifest.projectMcpServers + * + * — and NOTHING else. No other key is copied, stored, logged, or even retained + * past the parse: the parsed object is read for those two keys and dropped. + * + * Both directions go through the injected services: + * capture -> sanitizer.sanitizeJson (every leaf under env/environment/headers + * becomes {{REDACTED}}, unless includeSecrets) then templater + * .toTemplate over every string leaf, so an absolute command path + * folds to {{HOME}}/.... + * restore -> templater.fromTemplate over the serialized def, then a KEY-WISE + * merge into the target's ~/.claude.json. Keys arbella does not own + * are never touched, and a file that is not valid JSON is never + * overwritten. + * + * The restore side has exactly ONE decision function, `planMcpMerge`: it reads + * the target's current ~/.claude.json and reports which servers WOULD be written + * (plus the env keys those servers still need). `--dry-run` prints its actions, + * the post-restore reminder reads its `needsEnv`, and `restoreMcpServers` applies + * its decisions verbatim — so the three can never disagree about what a pull does. + * + * No direct node:fs, no clock. templateDeep/hydrateDef and the merge appliers are + * pure. + */ + +import type { CaptureContext, RestoreContext } from "../adapter.interface.js"; +import type { RestoreAction, SecretRef, ToolManifest } from "../../types.js"; + +import { REDACTED } from "../../core/sanitizer/patterns.js"; +import { isPlainObject, isUnsafeObjectKey } from "../../utils/object.js"; +import type { CommandRef } from "../../core/homefiles/scan.js"; +import { collectCommandRefs } from "../../core/homefiles/scan.js"; + +import { paths } from "./paths.js"; + +/** What a capture pass lifted out of ~/.claude.json. */ +export interface CapturedMcp { + /** name -> server definition (sanitized + templated), name-sorted. */ + mcpServers: ToolManifest["mcpServers"]; + /** Project-scope servers, sorted by (templated) project path. */ + projectMcpServers: ToolManifest["projectMcpServers"]; + /** SecretRefs for every value the sanitizer redacted. Metadata only. */ + secrets: SecretRef[]; + /** Non-fatal problems (unreadable / unparseable global state). */ + warnings: string[]; + /** + * Command references taken from the RAW (pre-sanitize, pre-template) server + * definitions, so `shared/home` capture can follow them to the launcher + * scripts they point at (`~/.local/bin/serena-mcp-start`). Built here because + * ~/.claude.json is parsed exactly once, and the templated defs in + * `mcpServers` above no longer contain machine paths to resolve. + */ + commandRefs: CommandRef[]; +} + +/** An empty result — used for "file absent" and "file unparseable" alike. */ +function emptyMcp(warnings: string[] = []): CapturedMcp { + return { + mcpServers: {}, + projectMcpServers: [], + secrets: [], + warnings, + commandRefs: [], + }; +} + +/* -------------------------------------------------------------------------- */ +/* Pure helpers */ +/* -------------------------------------------------------------------------- */ + +/** + * Map every STRING leaf of a JSON-ish value through `fn`, returning a new value. + * Objects and arrays are cloned; non-string scalars pass through untouched. + * Pure — the input is never mutated. + */ +export function templateDeep(value: T, fn: (s: string) => string): T { + if (typeof value === "string") return fn(value) as unknown as T; + if (Array.isArray(value)) { + return value.map((el) => templateDeep(el, fn)) as unknown as T; + } + if (isPlainObject(value)) { + const out: Record = {}; + for (const [k, v] of Object.entries(value)) { + // `out["__proto__"] = …` would rewrite this object's prototype instead of + // adding a field. A server definition read out of ~/.claude.json (or a + // backup repo) is parsed JSON, which CAN carry such a key. + if (isUnsafeObjectKey(k)) continue; + out[k] = templateDeep(v, fn); + } + return out as unknown as T; + } + return value; +} + +/** Longest launch summary an action description carries before it is elided. */ +export const MAX_LAUNCH_SUMMARY = 120; + +/** + * One line describing WHAT a server definition will start on this machine: + * ` ` for a spawned server, `` for an http/sse one. + * + * The dry run exists so a user can see what a pull will do BEFORE it does it, + * and "Register MCP server serena" hides the only part that matters — the + * command a backup repo just talked this machine into launching. Returns "" when + * the definition names neither (nothing honest to show), so the caller can fall + * back to the bare label. Truncated to {@link MAX_LAUNCH_SUMMARY} characters: + * this is a plan line, not a config dump. Pure. + */ +export function describeServerLaunch(def: Record): string { + const url = typeof def.url === "string" ? def.url.trim() : ""; + const transport = typeof def.type === "string" ? def.type.toLowerCase() : ""; + const remote = transport === "http" || transport === "sse" || transport === "streamable-http"; + if (url !== "" && (remote || typeof def.command !== "string")) return truncate(url); + + const command = typeof def.command === "string" ? def.command.trim() : ""; + if (command === "") return url === "" ? "" : truncate(url); + + const args = Array.isArray(def.args) + ? def.args.filter((a): a is string => typeof a === "string") + : []; + return truncate([command, ...args].join(" ").trim()); +} + +/** Cut `value` to {@link MAX_LAUNCH_SUMMARY} characters, marking the elision. */ +function truncate(value: string): string { + const collapsed = value.replace(/\s+/g, " ").trim(); + return collapsed.length <= MAX_LAUNCH_SUMMARY + ? collapsed + : `${collapsed.slice(0, MAX_LAUNCH_SUMMARY - 1)}…`; +} + +/** `