feat(phase2): wiki-as-SoT parallel sync scaffold

scripts/docs-sync-from-wiki.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+# docs-sync-from-wiki.sh — Phase 2 of the wiki-as-SoT migration.
+#
+# Reads from AEGIS wiki via a thin HTTP proxy (GET /api/wiki/:slug) and
+# writes Astro content collection files to src/content/docs/__from-wiki/.
+# Runs alongside the legacy docs-sync.sh — does NOT replace it. The output
+# directory is parallel so you can compare round-trip output against the
+# files produced by the legacy gh-API sync.
+#
+# Manifest: scripts/wiki-publish-manifest.json (slug → Astro page cosmetics).
+# Source: AEGIS wiki via $AEGIS_BASE/api/wiki/:slug — auth via AEGIS_TOKEN.
+#
+# Usage:
+#   AEGIS_TOKEN=... ./scripts/docs-sync-from-wiki.sh                  # sync all
+#   AEGIS_TOKEN=... ./scripts/docs-sync-from-wiki.sh --dry-run        # show what would change
+#   AEGIS_TOKEN=... ./scripts/docs-sync-from-wiki.sh --slug <slug>    # sync just one page
+#
+# Environment:
+#   AEGIS_TOKEN    — required; AEGIS HTTP API auth token
+#   AEGIS_BASE     — optional override of base URL (default: from manifest)
+#
+# See: AEGIS wiki `wiki-as-docs-sot-migration` for the full migration plan.
+# Phase 2 = this script + parallel output dir; later phases retire the legacy sync.
+#
+# REQUIRES: AEGIS HTTP wiki proxy endpoint (GET /api/wiki/:slug returning the
+# wiki_read shape: { page: { title, summary, body, last_verified, ... } }).
+# That endpoint is filed as a follow-up dependency on aegis-daemon. Until it
+# lands, the script will exit 1 with a 404 from AEGIS — proof-of-concept
+# files in the output dir are pre-generated for review.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+MANIFEST="${PROJECT_ROOT}/scripts/wiki-publish-manifest.json"
+
+DRY_RUN=false
+SLUG_FILTER=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --dry-run) DRY_RUN=true; shift ;;
+    --slug)    SLUG_FILTER="$2"; shift 2 ;;
+    *)         echo "Unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+log()  { echo "[$(date '+%H:%M:%S')] $*"; }
+ok()   { echo "[$(date '+%H:%M:%S')] + $*"; }
+skip() { echo "[$(date '+%H:%M:%S')] ~ $*"; }
+fail() { echo "[$(date '+%H:%M:%S')] ! $*" >&2; }
+
+if [[ ! -f "$MANIFEST" ]]; then
+  fail "Manifest not found: ${MANIFEST}"
+  exit 1
+fi
+
+if [[ -z "${AEGIS_TOKEN:-}" ]]; then
+  fail "AEGIS_TOKEN not set. Required for AEGIS HTTP API access."
+  exit 1
+fi
+
+AEGIS_BASE="${AEGIS_BASE:-$(python3 -c "import json; print(json.load(open('$MANIFEST'))['aegisBase'])")}"
+CONTENT_DIR="${PROJECT_ROOT}/$(python3 -c "import json; print(json.load(open('$MANIFEST'))['contentDir'])")"
+
+mkdir -p "$CONTENT_DIR"
+
+log "Wiki sync starting (Phase 2 parallel)"
+log "Manifest: $MANIFEST"
+log "Content dir: $CONTENT_DIR"
+log "AEGIS base: $AEGIS_BASE"
+$DRY_RUN && log "(dry-run mode — no files will be written)"
+
+SYNCED=0
+SKIPPED=0
+FAILED=0
+
+# Iterate manifest pages
+while IFS=$'\t' read -r slug page section order color tag; do
+
+  if [[ -n "$SLUG_FILTER" && "$slug" != "$SLUG_FILTER" ]]; then
+    continue
+  fi
+
+  log "Fetching wiki: $slug → $page"
+
+  response=$(curl -sS -m 15 \
+    -H "Authorization: Bearer $AEGIS_TOKEN" \
+    -w "\n__HTTP__%{http_code}" \
+    "${AEGIS_BASE}/api/wiki/${slug}" 2>&1) || {
+    fail "  curl failed for $slug"
+    FAILED=$((FAILED + 1))
+    continue
+  }
+
+  http_code=$(echo "$response" | tail -1 | sed 's/__HTTP__//')
+  body=$(echo "$response" | sed '$d')
+
+  if [[ "$http_code" != "200" ]]; then
+    fail "  HTTP $http_code from AEGIS for slug $slug"
+    fail "  body: $(echo "$body" | head -c 300)"
+    FAILED=$((FAILED + 1))
+    continue
+  fi
+
+  # Parse JSON: extract title, summary, body, last_verified
+  parsed=$(echo "$body" | python3 -c "
+import json, sys
+data = json.load(sys.stdin)
+page = data.get('page', {})
+print('---TITLE---')
+print(page.get('title', ''))
+print('---SUMMARY---')
+print(page.get('summary', ''))
+print('---LAST_VERIFIED---')
+print(page.get('last_verified', ''))
+print('---BODY---')
+print(page.get('body', ''))
+" 2>&1)
+
+  title=$(echo "$parsed" | sed -n '/^---TITLE---$/,/^---SUMMARY---$/p' | sed '1d;$d')
+  summary=$(echo "$parsed" | sed -n '/^---SUMMARY---$/,/^---LAST_VERIFIED---$/p' | sed '1d;$d')
+  last_verified=$(echo "$parsed" | sed -n '/^---LAST_VERIFIED---$/,/^---BODY---$/p' | sed '1d;$d')
+  page_body=$(echo "$parsed" | sed -n '/^---BODY---$/,$p' | sed '1d')
+
+  # Build Astro frontmatter from wiki metadata + manifest cosmetics
+  frontmatter=$(cat <<FRONTMATTER
+---
+title: "$(echo "$title" | sed 's/"/\\"/g')"
+description: "$(echo "$summary" | head -c 280 | sed 's/"/\\"/g' | tr '\n' ' ')"
+section: "$section"
+order: $order
+color: "$color"
+tag: "$tag"
+lastVerified: "$last_verified"
+sourceSlug: "$slug"
+---
+FRONTMATTER
+)
+
+  full_content="${frontmatter}
+
+${page_body}"
+
+  out_path="${CONTENT_DIR}/${page}"
+
+  # Compare with existing
+  if [[ -f "$out_path" ]]; then
+    existing_hash=$(md5sum "$out_path" | cut -d' ' -f1)
+    new_hash=$(echo "$full_content" | md5sum | cut -d' ' -f1)
+    if [[ "$existing_hash" == "$new_hash" ]]; then
+      skip "No changes: $page"
+      SKIPPED=$((SKIPPED + 1))
+      continue
+    fi
+  fi
+
+  if $DRY_RUN; then
+    ok "Would write: $page (from wiki:$slug)"
+  else
+    echo "$full_content" > "$out_path"
+    ok "Wrote: $page (from wiki:$slug, last_verified=$last_verified)"
+  fi
+  SYNCED=$((SYNCED + 1))
+
+done < <(python3 -c "
+import json
+m = json.load(open('$MANIFEST'))
+for p in m['pages']:
+    print(f\"{p['slug']}\t{p['page']}\t{p['section']}\t{p['order']}\t{p['color']}\t{p['tag']}\")
+")
+
+log ""
+log "Wiki sync complete: $SYNCED updated, $SKIPPED unchanged, $FAILED failed"
+$DRY_RUN && log "(dry-run — no files written)"
+
+exit $FAILED

scripts/docs-sync.sh

@@ -237,8 +237,12 @@ ${generated}"
   if [[ "$has_frontmatter" == "false" ]]; then
     existing="${CONTENT_DIR}/${page_key}"
     if [[ -f "$existing" ]]; then
-      # Extract existing frontmatter and prepend to new content
-      existing_fm=$(sed -n '/^---$/,/^---$/p' "$existing")
+      # Extract existing frontmatter (first --- block only) and prepend.
+      # awk single-state-machine: include lines starting with the first
+      # --- and stop after the second one. Robust against bodies that
+      # use ^---$ as markdown horizontal-rule separators (would otherwise
+      # double on each sync — see docs#19).
+      existing_fm=$(awk '/^---$/{n++; print; if(n==2)exit} n==1 && !/^---$/{print}' "$existing")
       content="${existing_fm}
 
 ${content}"

scripts/wiki-publish-manifest.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "Phase 2 wiki-as-SoT publish manifest — AEGIS wiki slugs → Astro page metadata. Per wiki-as-docs-sot-migration AEGIS decision.",
+  "version": 4,
+  "contentDir": "src/content/docs",
+  "_note": "Phase 2 parallel: page filenames are explicit per entry and use a -from-wiki suffix so they land at top-level (e.g., mcp-from-wiki.md) and route at /mcp-from-wiki for side-by-side comparison with the legacy /mcp page. Astro treats __-prefixed dirs as private and skips them, which is why we don't put files in a __from-wiki/ subdir. After Phase 4 migration (legacy retired), drop the suffix and use the actual page filename.",
+  "source": "aegis-wiki",
+  "aegisBase": "https://aegis.stackbilt.dev",
+  "pages": [
+    {
+      "slug": "mcp-gateway-architecture",
+      "page": "mcp-from-wiki.md",
+      "section": "platform",
+      "order": 5,
+      "color": "#22d3ee",
+      "tag": "05"
+    }
+  ]
+}

src/content/config.ts

@@ -9,6 +9,11 @@ const docs = defineCollection({
     order: z.number(),
     color: z.string(),
     tag: z.string(),
+    // Wiki-as-SoT migration (Phase 2): optional metadata surfaced from
+    // AEGIS wiki when the page is synced from there. Used to render a
+    // "Verified <date>" stamp and a back-link to the wiki source.
+    lastVerified: z.string().optional(),
+    sourceSlug: z.string().optional(),
   }),
 });
 

src/content/docs/mcp-from-wiki.md

@@ -0,0 +1,141 @@
+---
+title: "MCP Gateway — Upstream/Downstream Architecture"
+description: "Architecture map for Stackbilt-dev/stackbilt-mcp-gateway — the OAuth-authenticated Cloudflare Worker at mcp.stackbilt.dev/mcp that exposes the platform's product workers as a single MCP-compliant remote server."
+section: "platform"
+order: 5
+color: "#22d3ee"
+tag: "05"
+lastVerified: "2026-05-02"
+sourceSlug: "mcp-gateway-architecture"
+---
+
+## What this is
+
+The **Stackbilt MCP Gateway** is a Cloudflare Worker that exposes the platform's product backends as a single MCP-compliant remote server, OAuth-authenticated. It's the third leg of Stackbilder's three-consumer fractal: the same backend service bindings power the web UI on `stackbilder.com`, the Charter CLI, and the gateway. The gateway exists so MCP-native agents (Claude Code, Claude Desktop, custom MCP clients) get a single OAuth endpoint without taking a dependency on the web UI's session model.
+
+**Endpoint:** `https://mcp.stackbilt.dev/mcp`
+
+**Repo:** `Stackbilt-dev/stackbilt-mcp-gateway`
+
+**Stack:** Cloudflare Worker built on `@modelcontextprotocol/sdk` and `@cloudflare/workers-oauth-provider`. Listed on the official MCP registry at `registry.modelcontextprotocol.io`.
+
+## Topology
+
+```
+                          ┌────────────────────────┐
+                          │  AI agent / LM       │
+                          │  Claude Code,        │
+                          │  Claude Desktop,     │
+                          │  custom MCP clients  │
+                          └──────────┬───────────┘
+                                     │ OAuth + MCP (Streamable HTTP / SSE)
+                                     ▼
+                          ┌────────────────────────┐
+                          │  mcp.stackbilt.dev   │
+                          │  (gateway Worker)    │
+                          └──────────┬───────────┘
+                                     │ Service Bindings (in-colo, no HTTP)
+                                     ▼
+        ┌──────────────────────────────────────────┐
+        │  Backend product Workers (shared by all consumers) │
+        │   │                                                 │
+        │   ├─ edge-auth         (AUTH_SERVICE)               │
+        │   ├─ tarotscript-worker (TAROTSCRIPT, scaffold_*)    │
+        │   ├─ img-forge-mcp     (IMG_FORGE,    image_*)      │
+        │   ├─ stackbilt-engine  (ENGINE,       arch flows)   │
+        │   └─ stackbilt-deployer (DEPLOYER,    CF deploy)    │
+        └──────────────────────────────────────────┘
+                                     ▲
+                                     │ Service Bindings (parallel sibling consumers)
+        ┌──────────────────────────┐      ┌────────────────────┐
+        │  stackbilder.com         │      │  Charter CLI       │
+        │  (web UI + REST API)     │      │  (charter blast,   │
+        │  human users +           │      │   charter surface) │
+        │  ea_* API key callers    │      │  CI / local dev    │
+        └──────────────────────────┘      └────────────────────┘
+```
+
+**Key invariant:** the gateway and `stackbilder.com` are siblings, not parent/child. Both are CF Workers; both hold parallel service bindings to the same backend product workers; both route through `edge-auth` for entitlements + quota. A tenant's Pro tier is consistent across whichever consumer they use.
+
+## Service-binding map
+
+Authoritative source: `Stackbilt-dev/stackbilt-mcp-gateway/wrangler.toml`.
+
+| Binding | Backend Worker | Tool prefix on the gateway | What it routes |
+|---|---|---|---|
+| `AUTH_SERVICE` | `edge-auth` (entrypoint `AuthEntrypoint`) | (used internally) | Tenant resolution, API-key validation, OAuth grant storage in OAUTH_KV, entitlement checks, quota reservation |
+| `TAROTSCRIPT` | `tarotscript-worker` | `scaffold_*` | Deterministic project scaffolding, classification, GitHub publishing, CF deployment hand-off |
+| `IMG_FORGE` | `img-forge-mcp` | `image_*` | AI image generation (multi-provider, multi-tier) |
+| `ENGINE` | `stackbilt-engine` | (architecture flows) | Architecture mode pipeline (PRODUCT → SPRINT) — distinct from `tarotscript-worker`'s scaffold path |
+| `DEPLOYER` | `stackbilt-deployer` | (deploy flows) | Cloudflare Workers deployment, D1 provisioning, DNS via API |
+
+## Authentication
+
+| Method | Header | Use case |
+|---|---|---|
+| OAuth 2.1 + PKCE | `Authorization: Bearer <oauth-access-token>` | Recommended for end-user agent connections (Claude Desktop, Claude Code). Issued via `@cloudflare/workers-oauth-provider`. Tokens stored in `OAUTH_KV`. |
+| Static Bearer | `Authorization: Bearer <STACKBILT_MCP_TOKEN>` | Server-to-server / CI integrations. |
+
+The gateway intentionally does **not** accept `ea_*` API keys (those are issued from `stackbilder.com/settings` for the platform's REST API). Agent flows go through OAuth so the user can grant scoped consent.
+
+## Transports
+
+| Transport | Endpoint | Method | Use Case |
+|---|---|---|---|
+| **Streamable HTTP** | `/mcp` | POST | Modern MCP clients, single request/response |
+| **SSE Stream** | `/mcp` | GET | Server-pushed events, session-based |
+| **Server Info** | `/mcp/info` | GET | Capabilities discovery (no auth required) |
+
+Streamable HTTP sessions use the `Mcp-Session-Id` header. First `initialize` request returns a session ID; include it on subsequent requests; `DELETE /mcp` with the session ID to terminate.
+
+## Tool catalog state
+
+The gateway's tool surface is **migrating** at the time of this writing:
+
+- Legacy `flow_*` tools (architecture mode pipeline) — still bound, but the gateway repo's README marks them DEPRECATED and they're being replaced by deterministic `scaffold_*` tools (TarotScript-backed).
+- `scaffold_*` tools — the canonical replacement for `flow_*`. Faster (~20ms structure, ~2s with oracle prose), no LLM calls for file generation.
+- `image_*` tools — stable, route through `img-forge-mcp`.
+
+For the live tool catalog, query `tools/list` against the gateway directly, or read the gateway repo README. Do **not** treat any frozen tool listing as authoritative — the surface is in flux.
+
+## Documentation surface
+
+The canonical platform-side reference for the gateway is `Stackbilt-dev/stackbilt-web/docs/mcp.md` (rendered at `docs.stackbilt.dev/mcp`). The docs site sources this page from `stackbilt-web` per `docs-manifest.json` v3 (`Stackbilt-dev/docs@2bcd7cc`). The gateway repo's own `docs/mcp.md` is currently stale (still describes the decommissioned `stackbilt.dev/mcp` endpoint with Compass JWT auth and the deprecated `flow_*` tool catalog as canonical) — left for the gateway team to refresh.
+
+## What was deprecated
+
+This architecture replaces an earlier topology where:
+
+- The gateway lived behind `stackbilt.dev/mcp` (the `stackbilt.dev` domain now 301-redirects to `stackbilder.com`; `stackbilder.com/mcp/info` returns 404)
+- A third auth method, **Compass JWT**, was accepted (Compass was removed as a standalone product per `stackbilt-web/CLAUDE.md`; its governance logic is being absorbed into `stackbilt-engine`)
+- A separate **Compass Service Adapter (CSA)** transport mode existed for routing inside the platform (no longer relevant)
+- Cross-Compass unified-auth via `stackbilt.dev/api/auth/token` (endpoint dead)
+
+The `Stackbilt-dev/edgestack_v2` repo, which previously hosted the canonical `mcp.md` doc on its way to the docs site, is deprecated for documentation purposes and excluded from `docs-manifest.json` v3.
+
+## Two-consumer fractal in concrete form
+
+The `feedback_two_consumer_fractal` memory describes the principle: "Every product surface must work for both Human users AND LM agents. UI is one consumer of the API; MCP gateway + Charter CLI are siblings." The MCP gateway is the concrete instance:
+
+- Same backend product Workers as `stackbilder.com`
+- Same `AUTH_SERVICE` for entitlements
+- Same quota model
+- Different transport (MCP), different auth (OAuth), different consumer (LM agents)
+
+A new feature lands by adding a route on `stackbilder.com` (the canonical contract); the gateway gains parity by exposing it as an MCP tool that calls the same backend service binding. The web UI is the human surface; the gateway is the agent surface.
+
+## Authority
+
+- **Gateway repo:** `Stackbilt-dev/stackbilt-mcp-gateway`
+- **Wrangler config (binding map):** `Stackbilt-dev/stackbilt-mcp-gateway/wrangler.toml`
+- **Public registry:** `registry.modelcontextprotocol.io` (search: `stackbilt`)
+- **Docs site page:** `https://docs.stackbilt.dev/mcp`
+- **Canonical platform-side reference:** `Stackbilt-dev/stackbilt-web/docs/mcp.md`
+- **Manifest pointing here:** `Stackbilt-dev/docs/docs-manifest.json` v3
+- **Two-consumer fractal principle:** memory `feedback_two_consumer_fractal`
+
+When any of these contracts change shape (new service binding, new tool catalog, auth model change, transport addition, gateway URL change), update both the code and this page. Advance `last_verified`.
+
+---
+
+*Source: AEGIS wiki `mcp-gateway-architecture` · Verified 2026-05-02. This page is the Phase 2 round-trip POC for the wiki-as-docs-SoT migration. Once the AEGIS HTTP wiki proxy lands, `scripts/docs-sync-from-wiki.sh` will regenerate this file automatically.*