diff --git a/.github/workflows/static-quality.yml b/.github/workflows/static-quality.yml index c57c285..d3d4db2 100644 --- a/.github/workflows/static-quality.yml +++ b/.github/workflows/static-quality.yml @@ -19,10 +19,10 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: { node-version: 22 } - run: npm ci - # Cover the gap-analysis and atlas-harvest scripts too: they ship from - # scripts/ and were previously neither format-checked nor type-checked in - # CI (the other scripts/ files predate this gate and are out of scope here). - - run: npx prettier --check "src/**/*.ts" "scripts/gap-analysis/**/*.ts" "scripts/atlas-harvest.ts" + # Cover the gap-analysis script too: it ships from scripts/ and was + # previously neither format-checked nor type-checked in CI (the other + # scripts/ files predate this gate and are out of scope here). + - run: npx prettier --check "src/**/*.ts" "scripts/gap-analysis/**/*.ts" typecheck-scripts: runs-on: ubuntu-latest diff --git a/scripts/atlas-harvest/README.md b/scripts/atlas-harvest/README.md index 97f1ac3..6974446 100644 --- a/scripts/atlas-harvest/README.md +++ b/scripts/atlas-harvest/README.md @@ -1,9 +1,9 @@ # Atlas Harvest — running a harvest end-to-end This directory is the **Tier-1 leaf-fleet agent harness** for the Atlas seed -harvest. It is the *agent-orchestration half* of the system; the -*deterministic in-process half* lives in `src/atlas/**` and is driven by -`scripts/atlas-harvest.ts`. +harvest. It is the _agent-orchestration half_ of the system; the +_deterministic in-process half_ lives in `src/atlas/**` and is driven by +`src/atlas/harvest-cli.ts`. The two halves meet at one seam: **fragments on disk**. The leaf fleet writes one `CandidateFragment` JSON per unit into `runs//fragments/`; the @@ -12,21 +12,21 @@ driver reads them back and runs the deterministic Tiers 2-3 over the corpus. ``` SOURCES ──(Tier-1 leaf fleet: blitz agents, 1 unit each)──▶ runs//fragments/*.json │ - scripts/atlas-harvest.ts run + atlas harvest run (Tier-2 aggregate → classify → Tier-3 canonicalize → rag-dedup → validate) │ --upsert ▶ pending atlas_seed_entries rows │ - scripts/atlas-harvest.ts artifact + atlas harvest artifact │ Notion approval page (lead edits it) │ - scripts/atlas-harvest.ts sync + atlas harvest sync (checked & ¬excluded & approvable → approve; else → reject; 409 → conflicted) │ - scripts/atlas-harvest.ts reindex + atlas harvest reindex (AtlasDataProvider → pgvector) │ WIRE-ON (LAST, deferred — see below) @@ -35,22 +35,26 @@ SOURCES ──(Tier-1 leaf fleet: blitz agents, 1 unit each)──▶ runs/ +...`; the from-source equivalents are `npx tsx src/atlas/harvest-cli.ts + ...` (pre-build) and `node dist/atlas-cli.js harvest ...` + (post-build). - The seven adapters live in `src/atlas/adapters/` and are assembled into the `LeafAdapterRegistry` in exactly one place — `buildLeafAdapterRegistry()` in - `scripts/atlas-harvest.ts`. There is no shared `src/atlas/adapters/index.ts`. + `src/atlas/harvest-cli.ts`. There is no shared `src/atlas/adapters/index.ts`. --- ## The pieces -| Artifact | What it is | -|---|---| -| `blitz-manifest.md` | The source-sharded blitz decomposition for an actual harvest RUN — one shard per source family, each fanning out to tiny one-unit leaf tasks. | -| `leaf-prompt.md` | The per-leaf agent prompt TEMPLATE — handed ONE unit, builds the fragment the matching adapter would emit, writes exactly ONE fragment JSON. | -| `scripts/atlas-harvest.ts` | The in-process driver CLI (not in this dir — one level up). Runs Tiers 2-3, generates/syncs the Notion artifact, queues reindex. | +| Artifact | What it is | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `blitz-manifest.md` | The source-sharded blitz decomposition for an actual harvest RUN — one shard per source family, each fanning out to tiny one-unit leaf tasks. | +| `leaf-prompt.md` | The per-leaf agent prompt TEMPLATE — handed ONE unit, builds the fragment the matching adapter would emit, writes exactly ONE fragment JSON. | +| `src/atlas/harvest-cli.ts` | The in-process driver CLI (not in this dir — it lives in `src/atlas/`, mounted as `atlas harvest`). Runs Tiers 2-3, generates/syncs the Notion artifact, queues reindex. | --- @@ -76,7 +80,7 @@ the absolute fragments directory as inputs. ## Step 1 — Run the Tier-1 leaf fleet (this harness) The leaf fleet is launched as a `blitz` fleet from `blitz-manifest.md`. Each -slot is a *shard* over one source family (memory, PRs per repo, Notion, Linear, +slot is a _shard_ over one source family (memory, PRs per repo, Notion, Linear, episodic, source comments, showcase); each shard fans out to tiny leaf tasks, one **unit** per leaf. Every leaf: @@ -99,7 +103,7 @@ The output of this step is a directory of fragments. Nothing has touched the DB yet. > **Incremental ramp (org discipline).** Do NOT launch the full fleet on the -> first run. Start with ONE shard of ~5 units, run Step 2 as a `--dry-run`, +> first run. Start with ONE shard of ~4 units, run Step 2 as a `--dry-run`, > confirm the fragments parse, then ramp the shards up. (Serverless dry-runs > fail fast at 5 consecutive rag-probe failures — keep a serverless ramp at ≤4 > fragments or stub the search route.) See "Smoke-ramp" below. @@ -116,7 +120,7 @@ promoteValidation`, then (only with `--upsert`) writes each candidate as a Preview (writes NOTHING): ``` -npx tsx scripts/atlas-harvest.ts run \ +atlas harvest run \ --run-id \ --checkout \ --feature-registry @@ -125,7 +129,7 @@ npx tsx scripts/atlas-harvest.ts run \ Write pending rows: ``` -npx tsx scripts/atlas-harvest.ts run \ +atlas harvest run \ --run-id --upsert \ --checkout \ --feature-registry @@ -140,8 +144,12 @@ Required flags / env for `run` (enforced by the driver — it throws if missing) validation gate maps claims against to showcase-verify them. - `--token ` or `ANALYTICS_TOKEN` — bearer for the live endpoints; the rag-dedup gate probes `GET /api/search`. -- `--url ` or `PATHFINDER_BASE_URL` — the live Pathfinder base URL (default - `http://localhost:3001`). **A live server must be reachable** because the + +Base URL (NOT enforced — the driver warns and falls back if missing): + +- `--url ` or `PATHFINDER_BASE_URL` — the live Pathfinder base URL; when + neither is set the driver warns and falls back to `http://localhost:3001`. + **A live server must be reachable** because the rag-dedup gate makes one `search` round-trip per candidate (approximately: a candidate with too few distinct tokens to ever clear the overlap floor skips its probe entirely). @@ -152,7 +160,7 @@ pipeline but write NOTHING — overrides `--upsert`). Note that `--dry-run` still performs LIVE rag-dedup probes against the server — it skips the writes, not the probes. -The rag-dedup gate **never drops** a candidate; on corpus overlap it *marks* +The rag-dedup gate **never drops** a candidate; on corpus overlap it _marks_ the candidate (annotates `provenance.validated_against` + a `fused_from` evidence ref). The validation gate promotes `validation_status` (`unverified → source-verified → showcase-verified`) and marks a behavior / @@ -164,7 +172,7 @@ written; it just renders non-checkable in the approval artifact). ## Step 3 — Generate the Notion approval artifact ``` -npx tsx scripts/atlas-harvest.ts artifact \ +atlas harvest artifact \ --run-id \ --parent \ --checkout \ @@ -213,7 +221,7 @@ The edited page is the single source of truth for what gets ratified. ## Step 5 — Sync the edited page back to the DB ``` -npx tsx scripts/atlas-harvest.ts sync \ +atlas harvest sync \ --page \ --actor \ [--run-id ] @@ -233,7 +241,7 @@ server (row already settled / never existed) is treated as an idempotent no-op, so a re-run of `sync` is safe; those server-refused ratifications are tallied in a separate `conflicted` bucket rather than being counted as approved or rejected. Passing `--run-id` persists the run's final exclusion-rule SET into -its manifest so the *next* run's artifact can seed from it (omit it and the +its manifest so the _next_ run's artifact can seed from it (omit it and the driver warns that the rule set will NOT be persisted). The command prints ` approved, rejected, excluded-by-rule, conflicted`. @@ -241,15 +249,17 @@ driver warns that the rule set will NOT be persisted). The command prints An accidentally **indented** (Tab-nested) candidate checkbox is still discovered and enacted — the sync warns and asks you to un-indent it — but **rule bullets must remain top-level: an indented `atlas-rule:` bullet is not -parsed** — the sync warns about it and asks you to un-indent it, but the rule -stays out of enforcement and next-run seeding until you do. +parsed** — the sync warns about it (within the 3-level nested-scan cap the +sync descends; deeper nesting gets only a generic unscanned-children warning) +and asks you to un-indent it, but the rule stays out of enforcement and +next-run seeding until you do. --- ## Step 6 — Reindex ``` -npx tsx scripts/atlas-harvest.ts reindex [--scope full|source|repo] [--source ] [--repo ] +atlas harvest reindex [--scope full|source|repo] [--source ] [--repo ] ``` Requires `--token`/`ANALYTICS_TOKEN`. Queues a (scoped) reindex via @@ -279,14 +289,14 @@ empty/unapproved corpus serves nothing useful. Before launching the fleet, prove the fragment seam on a tiny ramp (the org's incremental-ramp discipline): -1. Hand-write ~3-5 valid `CandidateFragment` JSON files into a throwaway +1. Hand-write ~3-4 valid `CandidateFragment` JSON files into a throwaway `/tmp/atlas-smoke/_smoke/fragments/` (conform to `CandidateFragmentSchema` in `src/atlas/types.ts` — see the worked examples in `leaf-prompt.md`). 2. Dry-run the driver over them, pointing `--runs-dir` at the SAME throwaway root the fragments were written under: ``` - ANALYTICS_TOKEN=smoke npx tsx scripts/atlas-harvest.ts run \ + ANALYTICS_TOKEN=smoke atlas harvest run \ --run-id _smoke --runs-dir /tmp/atlas-smoke --dry-run \ --checkout fixtures/atlas/checkout \ --feature-registry fixtures/atlas/showcase/feature-registry.json @@ -294,7 +304,7 @@ incremental-ramp discipline): The summary line must report the number of fragment files you wrote — e.g. `atlas-harvest run [dry-run] run-id=_smoke: 3 fragments → 3 candidates → 0 - upserted` for 3 distinct fragments. A `0 fragments` line means the fragments +upserted` for 3 distinct fragments. A `0 fragments` line means the fragments directory and `--runs-dir` do not agree (the run read an empty/missing corpus) — the smoke pass is vacuous, fix the paths. diff --git a/scripts/atlas-harvest/blitz-manifest.md b/scripts/atlas-harvest/blitz-manifest.md index 46a3a2f..e6242ca 100644 --- a/scripts/atlas-harvest/blitz-manifest.md +++ b/scripts/atlas-harvest/blitz-manifest.md @@ -4,13 +4,13 @@ This is the `blitz` decomposition for an **actual harvest RUN** (not the codebase build — that is a different, already-shipped plan). It fans the Tier-1 acquisition out over the whole company's signal-bearing sources with maximal parallelism. The deterministic reduce/classify/validate half is NOT in -this fleet — it is the in-process driver (`scripts/atlas-harvest.ts run`) that +this fleet — it is the in-process driver (`atlas harvest run`) that runs AFTER the fleet, over the fragments this fleet produces. ## Shape - **Sharded by source family.** One shard per source family. A shard is a - *fan-out*: it enumerates its units and launches one tiny **leaf task** per + _fan-out_: it enumerates its units and launches one tiny **leaf task** per unit. Sharding by family keeps each leaf's adapter, MCP surface, and unit shape homogeneous, so the leaf prompt (`leaf-prompt.md`) is parameterized by family + unit, not rewritten per leaf. @@ -28,18 +28,23 @@ runs AFTER the fleet, over the fragments this fleet produces. ## Run parameters (every shard inherits these) -| Param | Meaning | -|---|---| -| `RUN_ID` | The run id (e.g. `2026-06-08-full`). All shards write under the same run. | -| `FRAGMENTS_DIR` | Absolute path to `runs//fragments/`. The single write target. | -| `AS_OF` | The harvest "as of" calendar date (`YYYY-MM-DD`) stamped into provenance freshness for sources that lack their own date. | +| Param | Meaning | +| --------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `RUN_ID` | The run id (e.g. `2026-06-08-full`). All shards write under the same run. | +| `FRAGMENTS_DIR` | Absolute path to `runs//fragments/`. The single write target. | +| `AS_OF` | The harvest "as of" calendar date (`YYYY-MM-DD`) stamped into provenance freshness for sources that lack their own date. | ## Fragment id convention Each leaf owns a unique, filesystem-safe, deterministic file stem so parallel -leaves can never collide. Writes are exclusive (`wx`) and FAIL LOUD on +leaves can never collide. The in-process RunStore writer +(`RunStore.writeFragment`) writes exclusively (`wx`) and FAILS LOUD on collision — a retried leaf must delete its prior fragment file first (or the -run must use a fresh run id), per the run-store error contract. Recommended: +run must use a fresh run id), per the run-store error contract. That guarantee +covers ONLY the in-process writer: leaves are out-of-process and write their +fragment files directly, so each leaf must create its file exclusively (fail +if it already exists — see leaf-prompt step 4), and unique stems remain the +fleet's primary collision defense. Recommended: `-` (e.g. `github-pr-pathfinder-1746`, `memory-feedback_nextjs_bundles_node_modules`, `notion-doc--` for the n-th decision split off a page). The id is the file stem only — the @@ -53,7 +58,7 @@ Each shard below names: the **adapter** whose contract its leaves emulate (in `src/atlas/adapters/` — leaves are out-of-process agents; the adapter source is the executable contract, not code a leaf invokes), the **registry sourcetype** key (as wired in `buildLeafAdapterRegistry()` in -`scripts/atlas-harvest.ts`), the **`*Unit`** shape each leaf assembles, the +`src/atlas/harvest-cli.ts`), the **`*Unit`** shape each leaf assembles, the **enumeration** that produces the units, and notes. ### Shard 1 — Memory store @@ -132,7 +137,10 @@ the **registry sourcetype** key (as wired in `buildLeafAdapterRegistry()` in (`distillEpisodicWindow`) — NOT a plain adapter call. - **Notes:** episodic knowledge is NEVER self-verifying — every fragment comes out `needsReview=true`, `validation_status="unverified"`, - `provenance_class="derived"` (the adapter re-asserts these). LLM calls go + `provenance_class="derived"`, `confidence="low"` (clamped — a stronger + distiller signal is an unsafe escalation), and `sensitivity` floored at + `"internal"` (any stronger distiller signal is preserved) — the adapter + re-asserts all of these. LLM calls go through the `OPENAI_BASE_URL` seam. Keep concurrency low (LLM + MCP). Split into sub-shards by session-date range if the window count is large. @@ -168,16 +176,16 @@ the **registry sourcetype** key (as wired in `buildLeafAdapterRegistry()` in ## Concurrency / scheduling summary -| Shard | Adapter | Registry key(s) | MCP-gated | LLM | Concurrency | -|---|---|---|---|---|---| -| 1 Memory | `memoryAdapter` | `memory` | no | no | high | -| 2 PRs (×3 repos) | `githubAdapter` | `github-pr` | no (gh/API) | no | high, per-repo bucket | -| 3 Issues | `githubAdapter` | `github-issue` | no (gh/API) | no | with repo bucket | -| 4 Notion | `notionAdapter` | `notion-doc` | yes (Notion) | no | low | -| 5 Linear | `linearAdapter` | `linear-doc` | yes (Linear) | no | low | -| 6 Episodic | `episodicAdapter` | `episodic` | yes (episodic) | **yes** | low | -| 7 Source-comment | `sourceCommentAdapter` | `agent-doc` | no | no | high | -| 8 Showcase | `showcaseAdapter` | `derived` | no | no | high | +| Shard | Adapter | Registry key(s) | MCP-gated | LLM | Concurrency | +| ---------------- | ---------------------- | --------------- | -------------- | ------- | --------------------- | +| 1 Memory | `memoryAdapter` | `memory` | no | no | high | +| 2 PRs (×3 repos) | `githubAdapter` | `github-pr` | no (gh/API) | no | high, per-repo bucket | +| 3 Issues | `githubAdapter` | `github-issue` | no (gh/API) | no | with repo bucket | +| 4 Notion | `notionAdapter` | `notion-doc` | yes (Notion) | no | low | +| 5 Linear | `linearAdapter` | `linear-doc` | yes (Linear) | no | low | +| 6 Episodic | `episodicAdapter` | `episodic` | yes (episodic) | **yes** | low | +| 7 Source-comment | `sourceCommentAdapter` | `agent-doc` | no | no | high | +| 8 Showcase | `showcaseAdapter` | `derived` | no | no | high | Global live-slot cap: **10** (org `blitz` ceiling). Pure shards (1, 2, 3, 7, 8) can run wide; MCP/LLM shards (4, 5, 6) run narrow to respect rate limits. @@ -190,7 +198,7 @@ reports DONE, the fragments dir is the handoff. The orchestrator then runs the driver over it: ``` -npx tsx scripts/atlas-harvest.ts run --run-id --upsert \ +atlas harvest run --run-id --upsert \ --checkout --feature-registry ``` @@ -201,8 +209,8 @@ See `README.md` for the full Steps 2-7 (run → artifact → edit → sync → r Do NOT launch all shards at full width on the first run. Ramp: -1. Run ONE shard (e.g. Memory) limited to ~5 units → ~5 fragments. -2. `npx tsx scripts/atlas-harvest.ts run --run-id --dry-run ...` and +1. Run ONE shard (e.g. Memory) limited to ~4 units → ~4 fragments. +2. `atlas harvest run --run-id --dry-run ...` and confirm the fragments parse (Zod) and Tiers 2-3 produce candidates (serverless dry-runs fail fast at 5 consecutive rag-probe failures — keep the serverless ramp at ≤4 fragments or stub the search route; see the @@ -210,5 +218,5 @@ Do NOT launch all shards at full width on the first run. Ramp: 3. Widen that shard, then add the next shard, re-running the dry-run gate each widening. -This catches a malformed-fragment / wrong-`*Unit`-shape defect at 5 units +This catches a malformed-fragment / wrong-`*Unit`-shape defect at ~4 units instead of after a thousand-unit fleet. diff --git a/scripts/atlas-harvest/leaf-prompt.md b/scripts/atlas-harvest/leaf-prompt.md index dbf9ca4..25cc2d5 100644 --- a/scripts/atlas-harvest/leaf-prompt.md +++ b/scripts/atlas-harvest/leaf-prompt.md @@ -49,10 +49,15 @@ STEPS - For episodic ONLY: use the LLM distill path (`distillEpisodicWindow`) to turn the window into the fragment, then HARD-SET the episodic invariants: `needsReview: true`, `validation_status: "unverified"`, - `provenance_class: "derived"`. + `provenance_class: "derived"`, `confidence: "low"` (clamped — a stronger + distiller signal is an unsafe escalation), and `sensitivity` floored at + `"internal"` (preserve any stronger distiller signal — e.g. `"secret"`/ + `"proprietary"` stays; only absent/weaker values become `"internal"`). 4. WRITE exactly ONE file: `/.json` containing the - single `CandidateFragment` object (pretty-printed JSON). Do NOT write more + single `CandidateFragment` object (pretty-printed JSON). Create it + EXCLUSIVELY — if the file already exists, STOP and report BLOCKED (stem + collision); never overwrite. Do NOT write more than one fragment per leaf. EXCEPTION: a single Notion page that records multiple ratified decisions splits into one fragment PER decision — in that case write `-1.json`, `-2.json`, … (still one page = one leaf). @@ -65,7 +70,7 @@ STEPS - evidence[] (kind-discriminated — see below), - validationTargets[] (symbols/paths the validation gate will grep). - 6. DO NOT touch the DB, DO NOT call scripts/atlas-harvest.ts, DO NOT read other + 6. DO NOT touch the DB, DO NOT call the `atlas harvest` driver, DO NOT read other leaves' fragments. Your only output is the one JSON file. REPORT @@ -189,7 +194,10 @@ These are the exact adapter input shapes (from `src/atlas/adapters/*.ts`). } ``` -**episodic** (`EpisodicWindowUnit`) — distill via the LLM, then hard-set the invariants: +**episodic** (`EpisodicWindowUnit`) — distill via the LLM, then hard-set the +invariants (`needsReview: true`, `validation_status: "unverified"`, +`provenance_class: "derived"`, `confidence: "low"` clamped, `sensitivity` +floored at `"internal"` preserving any stronger signal): ```jsonc { "convPath": "", "date": "2026-06-07", "text": "", "subsystem": "" } ``` diff --git a/src/__tests__/atlas-cli.test.ts b/src/__tests__/atlas-cli.test.ts index be4d84d..5a12d59 100644 --- a/src/__tests__/atlas-cli.test.ts +++ b/src/__tests__/atlas-cli.test.ts @@ -7,6 +7,7 @@ import { isAtlasCliEntrypoint, runAtlasCli, } from "../atlas-cli.js"; +import { runAtlasHarvestCli } from "../atlas/harvest-cli.js"; const PROJECT_ROOT = path.resolve(__dirname, "..", ".."); @@ -1314,3 +1315,156 @@ describe("atlas CLI", () => { } }); }); + +describe("atlas CLI — harvest verb (driver mount)", () => { + // The harvest driver (src/atlas/harvest-cli.ts) mounts as the `atlas harvest` + // subcommand: the remaining argv is forwarded verbatim to + // `runAtlasHarvestCli`, so `atlas harvest run --run-id ...` behaves exactly + // like the old standalone driver invocation (exit codes, stderr via + // formatCliError). These tests reach the harvest machinery through a cheap + // observable — its own commander/validation error text surfacing through the + // atlas binary — with no DB or network. + let stdout = ""; + let stderr = ""; + const io = { + stdout: (text: string) => { + stdout += text; + }, + stderr: (text: string) => { + stderr += text; + }, + }; + + afterEach(() => { + stdout = ""; + stderr = ""; + }); + + it("lists the harvest verb in the top-level help", async () => { + const exitCode = await runAtlasCli(["--help"], io); + + expect(exitCode).toBe(0); + expect(stdout).toContain("harvest"); + }); + + it("forwards argv to the harvest driver — its missing --run-id error surfaces through atlas", async () => { + const exitCode = await runAtlasCli(["harvest", "run"], io); + + expect(exitCode).not.toBe(0); + expect(stderr).toContain("--run-id"); + }); + + it("forwards option values intact — a parsed --run-id reaches the run command's own validation", async () => { + const exitCode = await runAtlasCli( + ["harvest", "run", "--run-id", "run-x"], + io, + ); + + // --run-id parsed by the harvest driver (its commander requiredOption is + // satisfied), so the failure is the NEXT gate: runCommand's own --checkout + // requirement, proving the forwarded argv ordering survived the mount. + expect(exitCode).toBe(1); + expect(stderr).not.toContain("--run-id "); + expect(stderr).toContain("--checkout"); + }); + + describe("mount fidelity — mounted tail matches the standalone driver byte-for-byte", () => { + // Parity harness: the SAME argv tail is fed to the mounted form + // (`atlas harvest `) and to the standalone driver + // (`runAtlasHarvestCli()`); exit code, stdout, and stderr must all + // be identical. This pins the mount contract: nothing in atlas-cli may + // consume or reorder ANY token of the tail — including a LEADING `--`, + // which a commander variadic `[args...]` would otherwise eat. + async function runBoth(tail: string[]) { + let mountedOut = ""; + let mountedErr = ""; + const mountedExit = await runAtlasCli(["harvest", ...tail], { + stdout: (text: string) => { + mountedOut += text; + }, + stderr: (text: string) => { + mountedErr += text; + }, + }); + + let standaloneOut = ""; + let standaloneErr = ""; + const standaloneExit = await runAtlasHarvestCli(tail, { + stdout: (text: string) => { + standaloneOut += text; + }, + stderr: (text: string) => { + standaloneErr += text; + }, + }); + + expect(mountedExit).toBe(standaloneExit); + expect(mountedOut).toBe(standaloneOut); + expect(mountedErr).toBe(standaloneErr); + return { + exitCode: standaloneExit, + stdout: standaloneOut, + stderr: standaloneErr, + }; + } + + it("preserves a LEADING `--` — `harvest -- --help` is an unknown command, not help", async () => { + const { exitCode, stderr } = await runBoth(["--", "--help"]); + + // Standalone, post-`--` tokens are operands: `--help` is an unknown + // command (exit 1), NOT a help request. + expect(exitCode).toBe(1); + expect(stderr).toContain("unknown command"); + }); + + it("preserves a LEADING `--` — `harvest -- run --run-id x` does NOT execute the run", async () => { + const { exitCode, stderr } = await runBoth([ + "--", + "run", + "--run-id", + "x", + ]); + + // Standalone, `--run-id x` after `--` are inert operands, so the run + // subcommand's requiredOption fails — the pipeline must NOT execute + // (no `--checkout` gate is ever reached). + expect(exitCode).toBe(1); + expect(stderr).toContain("--run-id "); + expect(stderr).not.toContain("--checkout"); + }); + + it("preserves a post-verb `--` — `harvest run -- --run-id x` keeps the operands inert", async () => { + const { exitCode, stderr } = await runBoth([ + "run", + "--", + "--run-id", + "x", + ]); + + expect(exitCode).toBe(1); + expect(stderr).toContain("--run-id "); + }); + + it("forwards a value-bearing pre-verb flag — `harvest --runs-dir /x run …` matches standalone", async () => { + const { exitCode, stderr } = await runBoth([ + "--runs-dir", + "/x", + "run", + "--run-id", + "y", + ]); + + // The driver's program level declares no --runs-dir option, so both + // forms reject it identically. + expect(exitCode).toBe(1); + expect(stderr).toContain("--runs-dir"); + }); + + it("shows the driver's own help — `harvest --help` exits 0 with the atlas-harvest usage", async () => { + const { exitCode, stdout } = await runBoth(["--help"]); + + expect(exitCode).toBe(0); + expect(stdout).toContain("Usage: atlas-harvest"); + }); + }); +}); diff --git a/src/__tests__/atlas-harvest-cli.test.ts b/src/__tests__/atlas-harvest-cli.test.ts index 122d93e..d011a07 100644 --- a/src/__tests__/atlas-harvest-cli.test.ts +++ b/src/__tests__/atlas-harvest-cli.test.ts @@ -1,6 +1,6 @@ // Harvest-driver CLI integration tests (plan S18 / §4 data-flow). // -// S18 is the DRIVER slot: `scripts/atlas-harvest.ts` is the SINGLE assembly +// S18 is the DRIVER slot: `src/atlas/harvest-cli.ts` is the SINGLE assembly // point for the leaf-adapter registry AND the in-process pipeline that turns a // run directory of CandidateFragment JSON files into `pending` atlas_seed_entries // rows. This suite drives the exported `runHarvest(opts)` directly (no @@ -78,7 +78,7 @@ import { formatCliError, runAtlasHarvestCli, type RunHarvestDeps, -} from "../../scripts/atlas-harvest.js"; +} from "../atlas/harvest-cli.js"; // The sync MODULE is mocked file-wide: the sync-summary CLI test below asserts // only the driver's output plumbing — sync's own enactment semantics live in diff --git a/src/__tests__/atlas-validate-checkout.test.ts b/src/__tests__/atlas-validate-checkout.test.ts index 38bb559..3043111 100644 --- a/src/__tests__/atlas-validate-checkout.test.ts +++ b/src/__tests__/atlas-validate-checkout.test.ts @@ -23,7 +23,7 @@ import { locateCheckoutDir, loadFeatureRegistry, } from "../atlas/validate-checkout.js"; -import { formatCliError } from "../../scripts/atlas-harvest.js"; +import { formatCliError } from "../atlas/harvest-cli.js"; function eaccesError(syscall: string): NodeJS.ErrnoException { return Object.assign( diff --git a/src/atlas-cli.ts b/src/atlas-cli.ts index 0a0b902..ff4df6f 100644 --- a/src/atlas-cli.ts +++ b/src/atlas-cli.ts @@ -4,6 +4,8 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { runAtlasHarvestCli } from "./atlas/harvest-cli.js"; + const DEFAULT_TOOL = "atlas-search"; const DEFAULT_FEEDBACK_TOOL = "submit-feedback"; const FEEDBACK_RATINGS = ["helpful", "not_helpful"] as const; @@ -425,11 +427,32 @@ export async function runAtlasCli( const writeOut = io.stdout ?? ((text: string) => process.stdout.write(text)); const writeErr = io.stderr ?? ((text: string) => process.stderr.write(text)); + // `harvest` short-circuits BEFORE commander parses: the raw tail is + // forwarded to the harvest driver genuinely verbatim — order intact, + // INCLUDING a leading `--`, which commander's `[args...]` variadic would + // otherwise consume as its own operand separator and silently drop, + // turning standalone-inert operands back into parsed options. The driver + // (src/atlas/harvest-cli.ts) owns its own commander program, io wiring, + // exit codes, and stderr formatting (formatCliError), so `atlas harvest + // run --run-id ...` behaves exactly like running the driver module + // directly. + if (argv[0] === "harvest") { + return runAtlasHarvestCli(argv.slice(1), { + stdout: writeOut, + stderr: writeErr, + }); + } + const program = new Command(); program .name("atlas") .description("Agent-facing Atlas search over Pathfinder MCP") .exitOverride() + // Required so the `harvest` mount below can use passThroughOptions(): + // option processing stops at the first subcommand, leaving each verb to + // parse its own flags (search/feedback already declare all their options + // locally, so their behavior is unchanged). + .enablePositionalOptions() .configureOutput({ writeOut, writeErr, @@ -470,9 +493,34 @@ export async function runAtlasCli( await feedback(query, options, writeOut); }); + // The harvest DRIVER (src/atlas/harvest-cli.ts) as a registered verb. + // Execution is handled by the pre-parse short-circuit at the top of + // runAtlasCli (which forwards the raw tail verbatim, leading `--` + // included), so this registration is UNREACHABLE for `atlas harvest ...` + // invocations. It is kept so `atlas --help` still lists the verb, and as a + // correct fallback for any commander-routed path. + let harvestExitCode: number | undefined; + program + .command("harvest") + .description( + "Atlas harvest driver — run the pipeline over a fragment corpus and " + + "drive ratification/index (subcommands: run, artifact, sync, reindex)", + ) + .helpOption(false) + .allowUnknownOption() + .allowExcessArguments(true) + .passThroughOptions() + .argument("[args...]", "Arguments forwarded to the harvest driver") + .action(async (args: string[]) => { + harvestExitCode = await runAtlasHarvestCli(args, { + stdout: writeOut, + stderr: writeErr, + }); + }); + try { await program.parseAsync(argv, { from: "user" }); - return 0; + return harvestExitCode ?? 0; } catch (error) { if (error instanceof CommanderError) { return error.exitCode; diff --git a/src/atlas/adapters/github.ts b/src/atlas/adapters/github.ts index 09d23f4..d7ce4d9 100644 --- a/src/atlas/adapters/github.ts +++ b/src/atlas/adapters/github.ts @@ -478,7 +478,7 @@ function extractIssue( export const githubAdapter: LeafAdapter = { // PRs and issues share one adapter; the fragment's own `sourcetype` field // distinguishes `github-pr` from `github-issue` per unit. The registry - // (`buildLeafAdapterRegistry` in scripts/atlas-harvest.ts) registers this + // (`buildLeafAdapterRegistry` in src/atlas/harvest-cli.ts) registers this // adapter object under BOTH keys; the declared `sourcetype` here is the PR // one (the dominant GitHub unit) per the LeafAdapter contract. sourcetype: "github-pr", diff --git a/src/atlas/adapters/types.ts b/src/atlas/adapters/types.ts index 230b654..1c5716e 100644 --- a/src/atlas/adapters/types.ts +++ b/src/atlas/adapters/types.ts @@ -4,7 +4,7 @@ // and the SHAPE of the adapter registry — but it deliberately does NOT assemble // the registry map. Per the plan (§2 / §4.2 / S2), the populated // `LeafAdapterRegistry` is built in exactly ONE place: the S18 harvest driver -// (`scripts/atlas-harvest.ts`), which imports all seven adapters. There is NO +// (`src/atlas/harvest-cli.ts`), which imports all seven adapters. There is NO // shared `src/atlas/adapters/index.ts`. S2 owns only the contract type and the // `getAdapter` accessor. // diff --git a/scripts/atlas-harvest.ts b/src/atlas/harvest-cli.ts similarity index 92% rename from scripts/atlas-harvest.ts rename to src/atlas/harvest-cli.ts index f0b1b69..aaa7bdf 100644 --- a/scripts/atlas-harvest.ts +++ b/src/atlas/harvest-cli.ts @@ -8,9 +8,12 @@ // shared `src/atlas/adapters/index.ts` (S3-S9 each own only their own adapter // file and never edit a shared index, which avoids 7-slot file contention). // -// NOTE: the existing `src/atlas-cli.ts` is the UNRELATED consumer-side Atlas -// retrieval CLI (agent-facing search over Pathfinder MCP). THIS file is the new -// harvest-side driver — different surface, no conflict, not edited here. +// NOTE: `src/atlas-cli.ts` is the consumer-side Atlas retrieval CLI +// (agent-facing search over Pathfinder MCP) — a different surface with its own +// env conventions. This driver now ALSO mounts there as the `atlas harvest` +// verb: atlas-cli forwards the remaining argv to `runAtlasHarvestCli`, so +// `atlas harvest run --run-id ...` behaves exactly like running this module +// directly (`npx tsx src/atlas/harvest-cli.ts run --run-id ...`). // // Pipeline (the spec §4 data-flow), per `run`: // @@ -24,10 +27,10 @@ // → toSeedEntryRow → upsertAtlasSeedCandidate (only when --upsert; --dry-run writes NOTHING) // // Subcommands: -// run --run-id [--upsert] [--dry-run] run the pipeline (preview / write pending rows) -// artifact --run-id --parent generate the Notion approval artifact -// sync --page --actor read the edited page → enact approve/reject -// reindex [--scope full|source|repo] [--source ] queue a (scoped) reindex +// run --run-id --checkout --feature-registry [--upsert] [--dry-run] run the pipeline (preview / write pending rows; needs --token|ANALYTICS_TOKEN) +// artifact --run-id --parent --checkout --feature-registry generate the Notion approval artifact (needs --notion-token|NOTION_TOKEN) +// sync --page --actor read the edited page → enact approve/reject (needs BOTH --token|ANALYTICS_TOKEN and --notion-token|NOTION_TOKEN) +// reindex [--scope full|source|repo] [--source ] [--repo ] queue a (scoped) reindex (needs --token|ANALYTICS_TOKEN) import fs from "node:fs"; import path from "node:path"; @@ -36,41 +39,35 @@ import { Command, CommanderError, Option } from "commander"; import { Client } from "@notionhq/client"; // ── The seven leaf adapters — imported HERE and nowhere else (assembly point) ── -import { memoryAdapter } from "../src/atlas/adapters/memory.js"; -import { githubAdapter } from "../src/atlas/adapters/github.js"; -import { notionAdapter } from "../src/atlas/adapters/notion.js"; -import { linearAdapter } from "../src/atlas/adapters/linear.js"; -import { episodicAdapter } from "../src/atlas/adapters/episodic.js"; -import { sourceCommentAdapter } from "../src/atlas/adapters/source-comment.js"; -import { showcaseAdapter } from "../src/atlas/adapters/showcase.js"; -import type { LeafAdapterRegistry } from "../src/atlas/adapters/types.js"; +import { memoryAdapter } from "./adapters/memory.js"; +import { githubAdapter } from "./adapters/github.js"; +import { notionAdapter } from "./adapters/notion.js"; +import { linearAdapter } from "./adapters/linear.js"; +import { episodicAdapter } from "./adapters/episodic.js"; +import { sourceCommentAdapter } from "./adapters/source-comment.js"; +import { showcaseAdapter } from "./adapters/showcase.js"; +import type { LeafAdapterRegistry } from "./adapters/types.js"; // ── Pipeline stages ──────────────────────────────────────────────────────────── -import { aggregate } from "../src/atlas/aggregate.js"; -import { finalizeClassification } from "../src/atlas/classify.js"; -import { canonicalize, recomputeRankScore } from "../src/atlas/canonicalize.js"; -import { - dedupAgainstRagCorpus, - type RagDedupContext, -} from "../src/atlas/rag-dedup.js"; -import { - promoteValidation, - type ValidationContext, -} from "../src/atlas/validate.js"; -import { loadValidationContext } from "../src/atlas/validate-checkout.js"; -import { toSeedEntryRow, type Candidate } from "../src/atlas/types.js"; +import { aggregate } from "./aggregate.js"; +import { finalizeClassification } from "./classify.js"; +import { canonicalize, recomputeRankScore } from "./canonicalize.js"; +import { dedupAgainstRagCorpus, type RagDedupContext } from "./rag-dedup.js"; +import { promoteValidation, type ValidationContext } from "./validate.js"; +import { loadValidationContext } from "./validate-checkout.js"; +import { toSeedEntryRow, type Candidate } from "./types.js"; import { RunStore, CorruptRunManifestError, type RunManifest, -} from "../src/atlas/run-store.js"; -import { AtlasHttpClient } from "../src/atlas/client.js"; -import { generateApprovalArtifact } from "../src/atlas/artifact/generate.js"; -import { syncApprovalArtifact } from "../src/atlas/artifact/sync.js"; -import { OpenAIDistiller, type LlmDistiller } from "../src/atlas/llm.js"; +} from "./run-store.js"; +import { AtlasHttpClient } from "./client.js"; +import { generateApprovalArtifact } from "./artifact/generate.js"; +import { syncApprovalArtifact } from "./artifact/sync.js"; +import { OpenAIDistiller, type LlmDistiller } from "./llm.js"; // ── Storage layer (EXISTING, origin/main) ────────────────────────────────────── -import { upsertAtlasSeedCandidate } from "../src/db/atlas.js"; +import { upsertAtlasSeedCandidate } from "../db/atlas.js"; // ── Registry assembly (THE single place the map is populated) ─────────────────── @@ -321,6 +318,7 @@ export function parseMinOverlap(raw: string): number { type WriteFn = (text: string) => void; +// NOTE: advisory console.warn output deliberately bypasses this injected io (it goes to process stderr). interface HarvestCliIo { stdout?: WriteFn; stderr?: WriteFn; diff --git a/tsconfig.json b/tsconfig.json index 5274dc6..7357262 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,11 +15,5 @@ "sourceMap": true }, "include": ["src/**/*"], - "exclude": [ - "node_modules", - "dist", - "scripts", - "src/__tests__/atlas-harvest-cli.test.ts", - "src/__tests__/atlas-validate-checkout.test.ts" - ] + "exclude": ["node_modules", "dist", "scripts"] } diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json index fcfd33c..7602826 100644 --- a/tsconfig.scripts.json +++ b/tsconfig.scripts.json @@ -4,11 +4,6 @@ "noEmit": true, "rootDir": null }, - "include": [ - "scripts/gap-analysis/**/*", - "scripts/atlas-harvest.ts", - "src/__tests__/atlas-harvest-cli.test.ts", - "src/__tests__/atlas-validate-checkout.test.ts" - ], + "include": ["scripts/gap-analysis/**/*"], "exclude": ["node_modules", "dist"] }