diff --git a/scripts/docs-sync-from-wiki.sh b/scripts/docs-sync-from-wiki.sh new file mode 100644 index 0000000..a04d56e --- /dev/null +++ b/scripts/docs-sync-from-wiki.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# docs-sync-from-wiki.sh — Phase 2 of the wiki-as-SoT migration. +# +# Reads from AEGIS wiki via a thin HTTP proxy (GET /api/wiki/:slug) and +# writes Astro content collection files to src/content/docs/__from-wiki/. +# Runs alongside the legacy docs-sync.sh — does NOT replace it. The output +# directory is parallel so you can compare round-trip output against the +# files produced by the legacy gh-API sync. +# +# Manifest: scripts/wiki-publish-manifest.json (slug → Astro page cosmetics). +# Source: AEGIS wiki via $AEGIS_BASE/api/wiki/:slug — auth via AEGIS_TOKEN. +# +# Usage: +# AEGIS_TOKEN=... ./scripts/docs-sync-from-wiki.sh # sync all +# AEGIS_TOKEN=... ./scripts/docs-sync-from-wiki.sh --dry-run # show what would change +# AEGIS_TOKEN=... ./scripts/docs-sync-from-wiki.sh --slug # sync just one page +# +# Environment: +# AEGIS_TOKEN — required; AEGIS HTTP API auth token +# AEGIS_BASE — optional override of base URL (default: from manifest) +# +# See: AEGIS wiki `wiki-as-docs-sot-migration` for the full migration plan. +# Phase 2 = this script + parallel output dir; later phases retire the legacy sync. +# +# REQUIRES: AEGIS HTTP wiki proxy endpoint (GET /api/wiki/:slug returning the +# wiki_read shape: { page: { title, summary, body, last_verified, ... } }). +# That endpoint is filed as a follow-up dependency on aegis-daemon. Until it +# lands, the script will exit 1 with a 404 from AEGIS — proof-of-concept +# files in the output dir are pre-generated for review. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MANIFEST="${PROJECT_ROOT}/scripts/wiki-publish-manifest.json" + +DRY_RUN=false +SLUG_FILTER="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; shift ;; + --slug) SLUG_FILTER="$2"; shift 2 ;; + *) echo "Unknown arg: $1" >&2; exit 1 ;; + esac +done + +log() { echo "[$(date '+%H:%M:%S')] $*"; } +ok() { echo "[$(date '+%H:%M:%S')] + $*"; } +skip() { echo "[$(date '+%H:%M:%S')] ~ $*"; } +fail() { echo "[$(date '+%H:%M:%S')] ! $*" >&2; } + +if [[ ! -f "$MANIFEST" ]]; then + fail "Manifest not found: ${MANIFEST}" + exit 1 +fi + +if [[ -z "${AEGIS_TOKEN:-}" ]]; then + fail "AEGIS_TOKEN not set. Required for AEGIS HTTP API access." + exit 1 +fi + +AEGIS_BASE="${AEGIS_BASE:-$(python3 -c "import json; print(json.load(open('$MANIFEST'))['aegisBase'])")}" +CONTENT_DIR="${PROJECT_ROOT}/$(python3 -c "import json; print(json.load(open('$MANIFEST'))['contentDir'])")" + +mkdir -p "$CONTENT_DIR" + +log "Wiki sync starting (Phase 2 parallel)" +log "Manifest: $MANIFEST" +log "Content dir: $CONTENT_DIR" +log "AEGIS base: $AEGIS_BASE" +$DRY_RUN && log "(dry-run mode — no files will be written)" + +SYNCED=0 +SKIPPED=0 +FAILED=0 + +# Iterate manifest pages +while IFS=$'\t' read -r slug page section order color tag; do + + if [[ -n "$SLUG_FILTER" && "$slug" != "$SLUG_FILTER" ]]; then + continue + fi + + log "Fetching wiki: $slug → $page" + + response=$(curl -sS -m 15 \ + -H "Authorization: Bearer $AEGIS_TOKEN" \ + -w "\n__HTTP__%{http_code}" \ + "${AEGIS_BASE}/api/wiki/${slug}" 2>&1) || { + fail " curl failed for $slug" + FAILED=$((FAILED + 1)) + continue + } + + http_code=$(echo "$response" | tail -1 | sed 's/__HTTP__//') + body=$(echo "$response" | sed '$d') + + if [[ "$http_code" != "200" ]]; then + fail " HTTP $http_code from AEGIS for slug $slug" + fail " body: $(echo "$body" | head -c 300)" + FAILED=$((FAILED + 1)) + continue + fi + + # Parse JSON: extract title, summary, body, last_verified + parsed=$(echo "$body" | python3 -c " +import json, sys +data = json.load(sys.stdin) +page = data.get('page', {}) +print('---TITLE---') +print(page.get('title', '')) +print('---SUMMARY---') +print(page.get('summary', '')) +print('---LAST_VERIFIED---') +print(page.get('last_verified', '')) +print('---BODY---') +print(page.get('body', '')) +" 2>&1) + + title=$(echo "$parsed" | sed -n '/^---TITLE---$/,/^---SUMMARY---$/p' | sed '1d;$d') + summary=$(echo "$parsed" | sed -n '/^---SUMMARY---$/,/^---LAST_VERIFIED---$/p' | sed '1d;$d') + last_verified=$(echo "$parsed" | sed -n '/^---LAST_VERIFIED---$/,/^---BODY---$/p' | sed '1d;$d') + page_body=$(echo "$parsed" | sed -n '/^---BODY---$/,$p' | sed '1d') + + # Build Astro frontmatter from wiki metadata + manifest cosmetics + frontmatter=$(cat < "$out_path" + ok "Wrote: $page (from wiki:$slug, last_verified=$last_verified)" + fi + SYNCED=$((SYNCED + 1)) + +done < <(python3 -c " +import json +m = json.load(open('$MANIFEST')) +for p in m['pages']: + print(f\"{p['slug']}\t{p['page']}\t{p['section']}\t{p['order']}\t{p['color']}\t{p['tag']}\") +") + +log "" +log "Wiki sync complete: $SYNCED updated, $SKIPPED unchanged, $FAILED failed" +$DRY_RUN && log "(dry-run — no files written)" + +exit $FAILED diff --git a/scripts/docs-sync.sh b/scripts/docs-sync.sh index 9638ba0..076e9b3 100644 --- a/scripts/docs-sync.sh +++ b/scripts/docs-sync.sh @@ -237,8 +237,12 @@ ${generated}" if [[ "$has_frontmatter" == "false" ]]; then existing="${CONTENT_DIR}/${page_key}" if [[ -f "$existing" ]]; then - # Extract existing frontmatter and prepend to new content - existing_fm=$(sed -n '/^---$/,/^---$/p' "$existing") + # Extract existing frontmatter (first --- block only) and prepend. + # awk single-state-machine: include lines starting with the first + # --- and stop after the second one. Robust against bodies that + # use ^---$ as markdown horizontal-rule separators (would otherwise + # double on each sync — see docs#19). + existing_fm=$(awk '/^---$/{n++; print; if(n==2)exit} n==1 && !/^---$/{print}' "$existing") content="${existing_fm} ${content}" diff --git a/scripts/wiki-publish-manifest.json b/scripts/wiki-publish-manifest.json new file mode 100644 index 0000000..c6e268c --- /dev/null +++ b/scripts/wiki-publish-manifest.json @@ -0,0 +1,18 @@ +{ + "$schema": "Phase 2 wiki-as-SoT publish manifest — AEGIS wiki slugs → Astro page metadata. Per wiki-as-docs-sot-migration AEGIS decision.", + "version": 4, + "contentDir": "src/content/docs", + "_note": "Phase 2 parallel: page filenames are explicit per entry and use a -from-wiki suffix so they land at top-level (e.g., mcp-from-wiki.md) and route at /mcp-from-wiki for side-by-side comparison with the legacy /mcp page. Astro treats __-prefixed dirs as private and skips them, which is why we don't put files in a __from-wiki/ subdir. After Phase 4 migration (legacy retired), drop the suffix and use the actual page filename.", + "source": "aegis-wiki", + "aegisBase": "https://aegis.stackbilt.dev", + "pages": [ + { + "slug": "mcp-gateway-architecture", + "page": "mcp-from-wiki.md", + "section": "platform", + "order": 5, + "color": "#22d3ee", + "tag": "05" + } + ] +} diff --git a/src/content/config.ts b/src/content/config.ts index 7f03c52..0336f24 100644 --- a/src/content/config.ts +++ b/src/content/config.ts @@ -9,6 +9,11 @@ const docs = defineCollection({ order: z.number(), color: z.string(), tag: z.string(), + // Wiki-as-SoT migration (Phase 2): optional metadata surfaced from + // AEGIS wiki when the page is synced from there. Used to render a + // "Verified " stamp and a back-link to the wiki source. + lastVerified: z.string().optional(), + sourceSlug: z.string().optional(), }), }); diff --git a/src/content/docs/mcp-from-wiki.md b/src/content/docs/mcp-from-wiki.md new file mode 100644 index 0000000..305c925 --- /dev/null +++ b/src/content/docs/mcp-from-wiki.md @@ -0,0 +1,141 @@ +--- +title: "MCP Gateway — Upstream/Downstream Architecture" +description: "Architecture map for Stackbilt-dev/stackbilt-mcp-gateway — the OAuth-authenticated Cloudflare Worker at mcp.stackbilt.dev/mcp that exposes the platform's product workers as a single MCP-compliant remote server." +section: "platform" +order: 5 +color: "#22d3ee" +tag: "05" +lastVerified: "2026-05-02" +sourceSlug: "mcp-gateway-architecture" +--- + +## What this is + +The **Stackbilt MCP Gateway** is a Cloudflare Worker that exposes the platform's product backends as a single MCP-compliant remote server, OAuth-authenticated. It's the third leg of Stackbilder's three-consumer fractal: the same backend service bindings power the web UI on `stackbilder.com`, the Charter CLI, and the gateway. The gateway exists so MCP-native agents (Claude Code, Claude Desktop, custom MCP clients) get a single OAuth endpoint without taking a dependency on the web UI's session model. + +**Endpoint:** `https://mcp.stackbilt.dev/mcp` + +**Repo:** `Stackbilt-dev/stackbilt-mcp-gateway` + +**Stack:** Cloudflare Worker built on `@modelcontextprotocol/sdk` and `@cloudflare/workers-oauth-provider`. Listed on the official MCP registry at `registry.modelcontextprotocol.io`. + +## Topology + +``` + ┌────────────────────────┐ + │ AI agent / LM │ + │ Claude Code, │ + │ Claude Desktop, │ + │ custom MCP clients │ + └──────────┬───────────┘ + │ OAuth + MCP (Streamable HTTP / SSE) + ▼ + ┌────────────────────────┐ + │ mcp.stackbilt.dev │ + │ (gateway Worker) │ + └──────────┬───────────┘ + │ Service Bindings (in-colo, no HTTP) + ▼ + ┌──────────────────────────────────────────┐ + │ Backend product Workers (shared by all consumers) │ + │ │ │ + │ ├─ edge-auth (AUTH_SERVICE) │ + │ ├─ tarotscript-worker (TAROTSCRIPT, scaffold_*) │ + │ ├─ img-forge-mcp (IMG_FORGE, image_*) │ + │ ├─ stackbilt-engine (ENGINE, arch flows) │ + │ └─ stackbilt-deployer (DEPLOYER, CF deploy) │ + └──────────────────────────────────────────┘ + ▲ + │ Service Bindings (parallel sibling consumers) + ┌──────────────────────────┐ ┌────────────────────┐ + │ stackbilder.com │ │ Charter CLI │ + │ (web UI + REST API) │ │ (charter blast, │ + │ human users + │ │ charter surface) │ + │ ea_* API key callers │ │ CI / local dev │ + └──────────────────────────┘ └────────────────────┘ +``` + +**Key invariant:** the gateway and `stackbilder.com` are siblings, not parent/child. Both are CF Workers; both hold parallel service bindings to the same backend product workers; both route through `edge-auth` for entitlements + quota. A tenant's Pro tier is consistent across whichever consumer they use. + +## Service-binding map + +Authoritative source: `Stackbilt-dev/stackbilt-mcp-gateway/wrangler.toml`. + +| Binding | Backend Worker | Tool prefix on the gateway | What it routes | +|---|---|---|---| +| `AUTH_SERVICE` | `edge-auth` (entrypoint `AuthEntrypoint`) | (used internally) | Tenant resolution, API-key validation, OAuth grant storage in OAUTH_KV, entitlement checks, quota reservation | +| `TAROTSCRIPT` | `tarotscript-worker` | `scaffold_*` | Deterministic project scaffolding, classification, GitHub publishing, CF deployment hand-off | +| `IMG_FORGE` | `img-forge-mcp` | `image_*` | AI image generation (multi-provider, multi-tier) | +| `ENGINE` | `stackbilt-engine` | (architecture flows) | Architecture mode pipeline (PRODUCT → SPRINT) — distinct from `tarotscript-worker`'s scaffold path | +| `DEPLOYER` | `stackbilt-deployer` | (deploy flows) | Cloudflare Workers deployment, D1 provisioning, DNS via API | + +## Authentication + +| Method | Header | Use case | +|---|---|---| +| OAuth 2.1 + PKCE | `Authorization: Bearer ` | Recommended for end-user agent connections (Claude Desktop, Claude Code). Issued via `@cloudflare/workers-oauth-provider`. Tokens stored in `OAUTH_KV`. | +| Static Bearer | `Authorization: Bearer ` | Server-to-server / CI integrations. | + +The gateway intentionally does **not** accept `ea_*` API keys (those are issued from `stackbilder.com/settings` for the platform's REST API). Agent flows go through OAuth so the user can grant scoped consent. + +## Transports + +| Transport | Endpoint | Method | Use Case | +|---|---|---|---| +| **Streamable HTTP** | `/mcp` | POST | Modern MCP clients, single request/response | +| **SSE Stream** | `/mcp` | GET | Server-pushed events, session-based | +| **Server Info** | `/mcp/info` | GET | Capabilities discovery (no auth required) | + +Streamable HTTP sessions use the `Mcp-Session-Id` header. First `initialize` request returns a session ID; include it on subsequent requests; `DELETE /mcp` with the session ID to terminate. + +## Tool catalog state + +The gateway's tool surface is **migrating** at the time of this writing: + +- Legacy `flow_*` tools (architecture mode pipeline) — still bound, but the gateway repo's README marks them DEPRECATED and they're being replaced by deterministic `scaffold_*` tools (TarotScript-backed). +- `scaffold_*` tools — the canonical replacement for `flow_*`. Faster (~20ms structure, ~2s with oracle prose), no LLM calls for file generation. +- `image_*` tools — stable, route through `img-forge-mcp`. + +For the live tool catalog, query `tools/list` against the gateway directly, or read the gateway repo README. Do **not** treat any frozen tool listing as authoritative — the surface is in flux. + +## Documentation surface + +The canonical platform-side reference for the gateway is `Stackbilt-dev/stackbilt-web/docs/mcp.md` (rendered at `docs.stackbilt.dev/mcp`). The docs site sources this page from `stackbilt-web` per `docs-manifest.json` v3 (`Stackbilt-dev/docs@2bcd7cc`). The gateway repo's own `docs/mcp.md` is currently stale (still describes the decommissioned `stackbilt.dev/mcp` endpoint with Compass JWT auth and the deprecated `flow_*` tool catalog as canonical) — left for the gateway team to refresh. + +## What was deprecated + +This architecture replaces an earlier topology where: + +- The gateway lived behind `stackbilt.dev/mcp` (the `stackbilt.dev` domain now 301-redirects to `stackbilder.com`; `stackbilder.com/mcp/info` returns 404) +- A third auth method, **Compass JWT**, was accepted (Compass was removed as a standalone product per `stackbilt-web/CLAUDE.md`; its governance logic is being absorbed into `stackbilt-engine`) +- A separate **Compass Service Adapter (CSA)** transport mode existed for routing inside the platform (no longer relevant) +- Cross-Compass unified-auth via `stackbilt.dev/api/auth/token` (endpoint dead) + +The `Stackbilt-dev/edgestack_v2` repo, which previously hosted the canonical `mcp.md` doc on its way to the docs site, is deprecated for documentation purposes and excluded from `docs-manifest.json` v3. + +## Two-consumer fractal in concrete form + +The `feedback_two_consumer_fractal` memory describes the principle: "Every product surface must work for both Human users AND LM agents. UI is one consumer of the API; MCP gateway + Charter CLI are siblings." The MCP gateway is the concrete instance: + +- Same backend product Workers as `stackbilder.com` +- Same `AUTH_SERVICE` for entitlements +- Same quota model +- Different transport (MCP), different auth (OAuth), different consumer (LM agents) + +A new feature lands by adding a route on `stackbilder.com` (the canonical contract); the gateway gains parity by exposing it as an MCP tool that calls the same backend service binding. The web UI is the human surface; the gateway is the agent surface. + +## Authority + +- **Gateway repo:** `Stackbilt-dev/stackbilt-mcp-gateway` +- **Wrangler config (binding map):** `Stackbilt-dev/stackbilt-mcp-gateway/wrangler.toml` +- **Public registry:** `registry.modelcontextprotocol.io` (search: `stackbilt`) +- **Docs site page:** `https://docs.stackbilt.dev/mcp` +- **Canonical platform-side reference:** `Stackbilt-dev/stackbilt-web/docs/mcp.md` +- **Manifest pointing here:** `Stackbilt-dev/docs/docs-manifest.json` v3 +- **Two-consumer fractal principle:** memory `feedback_two_consumer_fractal` + +When any of these contracts change shape (new service binding, new tool catalog, auth model change, transport addition, gateway URL change), update both the code and this page. Advance `last_verified`. + +--- + +*Source: AEGIS wiki `mcp-gateway-architecture` · Verified 2026-05-02. This page is the Phase 2 round-trip POC for the wiki-as-docs-SoT migration. Once the AEGIS HTTP wiki proxy lands, `scripts/docs-sync-from-wiki.sh` will regenerate this file automatically.*